fix(hands,jobs): watchdog window reads the room tail, not the pending cursor

This commit is contained in:
hh committed 2026-09-23 18:18:27 +00:00
1 parent 91bd50a66d
commit 47c31ac356
6 files changed
+341 -37

No files matched your search

+158 -21
View File
@@ -1,6 +1,8 @@
import logging
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from typing import Any, cast
from urllib.parse import parse_qsl, urlsplit
from zoneinfo import ZoneInfo
from beaver_agent.hands.heartbeat import (
@@ -14,7 +16,7 @@ from beaver_agent.hands.heartbeat import (
parse_span,
span,
)
from beaver_agent.hands.vibegram import Event
from beaver_agent.hands.vibegram import PAGE, Event, Vibegram, Window
WARSAW = ZoneInfo("Europe/Warsaw")
NOW = datetime(2026, 9, 22, 15, 42, tzinfo=UTC)
@@ -100,7 +102,9 @@ def test_beat_line_says_unknown_instead_of_guessing():
def test_fresh_beat_arms_and_stays_quiet(tmp_path):
w = state(tmp_path)
report = look(w, [hb(NOW - timedelta(hours=10))], now=NOW, res=timedelta(hours=24))
report = look(
w, [hb(NOW - timedelta(hours=10))], now=NOW, res=timedelta(hours=24), full=True
)
assert report.armed and report.alarm is None and not report.stale
assert report.threshold == timedelta(hours=36)
assert "armed=yes" in report.ack and "win=25" in report.ack
@@ -111,19 +115,21 @@ def test_fresh_beat_arms_and_stays_quiet(tmp_path):
def test_threshold_is_every_times_one_and_a_half(tmp_path):
# 30 часов при every=24h - ещё не тревога: одна потерянная строка не в счёт
quiet = state(tmp_path, seen=NOW - timedelta(hours=30), every=timedelta(hours=24))
assert look(quiet, [], now=NOW, res=timedelta(hours=24)).alarm is None
assert look(quiet, [], now=NOW, res=timedelta(hours=24), full=True).alarm is None
# 37 часов - уже за порогом 36
loud = state(tmp_path, seen=NOW - timedelta(hours=37), every=timedelta(hours=24))
report = look(loud, [], now=NOW, res=timedelta(hours=24))
report = look(loud, [], now=NOW, res=timedelta(hours=24), full=True)
assert report.alarm == "silence"
# Порог берётся из строки соседа, а не из нашей константы
slow = state(tmp_path, seen=NOW - timedelta(hours=37), every=timedelta(hours=48))
assert look(slow, [], now=NOW, res=timedelta(hours=24)).alarm is None
assert look(slow, [], now=NOW, res=timedelta(hours=24), full=True).alarm is None
def test_last_beat_wins_over_gaps(tmp_path):
w = state(tmp_path, seen=NOW - timedelta(hours=40), every=timedelta(hours=24))
report = look(w, [hb(NOW - timedelta(hours=1), eid=120)], now=NOW, res=None)
report = look(
w, [hb(NOW - timedelta(hours=1), eid=120)], now=NOW, res=None, full=True
)
assert report.alarm is None
assert report.age is not None and report.age < timedelta(hours=2)
assert "res=unknown" in report.ack
@@ -131,7 +137,7 @@ def test_last_beat_wins_over_gaps(tmp_path):
def test_unknown_every_falls_back_to_last_known_and_says_so(tmp_path):
w = state(tmp_path, every=timedelta(hours=24))
report = look(w, [hb(NOW, every="unknown", eid=130)], now=NOW, res=None)
report = look(w, [hb(NOW, every="unknown", eid=130)], now=NOW, res=None, full=True)
assert report.threshold == timedelta(hours=36)
assert "thr=36h-fallback" in report.ack
assert w.every == timedelta(hours=24)
@@ -139,7 +145,7 @@ def test_unknown_every_falls_back_to_last_known_and_says_so(tmp_path):
def test_no_period_ever_means_no_measurement_and_no_alarm(tmp_path):
w = state(tmp_path, seen=NOW - timedelta(days=9))
report = look(w, [], now=NOW, res=timedelta(hours=24))
report = look(w, [], now=NOW, res=timedelta(hours=24), full=True)
assert report.alarm is None and report.threshold is None
assert "thr=unknown" in report.ack
assert "мерить нечем" in report.note
@@ -147,7 +153,7 @@ def test_no_period_ever_means_no_measurement_and_no_alarm(tmp_path):
def test_never_seen_is_armed_no_not_alarm(tmp_path):
w = state(tmp_path)
report = look(w, [], now=NOW, res=timedelta(hours=24))
report = look(w, [], now=NOW, res=timedelta(hours=24), full=True)
assert not report.armed and report.alarm is None
assert "armed=no" in report.ack and "last=never seen=never" in report.ack
assert w.unarmed_since == NOW
@@ -155,11 +161,11 @@ def test_never_seen_is_armed_no_not_alarm(tmp_path):
def test_seven_days_unarmed_tells_the_human_once(tmp_path):
w = state(tmp_path, unarmed_since=NOW - timedelta(days=7))
assert look(w, [], now=NOW, res=timedelta(hours=24)).alarm == "unarmed"
assert look(w, [], now=NOW, res=timedelta(hours=24), full=True).alarm == "unarmed"
w.unarmed_told = True
assert look(w, [], now=NOW, res=timedelta(hours=24)).alarm is None
assert look(w, [], now=NOW, res=timedelta(hours=24), full=True).alarm is None
# Пришла первая отметка - кольцо вооружилось, счётчик забыт
look(w, [hb(NOW, eid=140)], now=NOW, res=timedelta(hours=24))
look(w, [hb(NOW, eid=140)], now=NOW, res=timedelta(hours=24), full=True)
assert w.unarmed_since is None
@@ -167,12 +173,16 @@ def test_stale_window_forbids_alarm(tmp_path):
w = state(
tmp_path, seen=NOW - timedelta(hours=40), every=timedelta(hours=24), top=500
)
report = look(w, [hb(NOW - timedelta(hours=40), eid=7)], now=NOW, res=None)
report = look(
w, [hb(NOW - timedelta(hours=40), eid=7)], now=NOW, res=None, full=True
)
assert report.stale and report.alarm is None
assert "win=stale" in report.ack
assert w.top == 500 # съехавшее окно не двигает отметку максимума
# Окно снова вперёд - тревога берётся
fresh = look(w, [hb(NOW - timedelta(hours=40), eid=501)], now=NOW, res=None)
fresh = look(
w, [hb(NOW - timedelta(hours=40), eid=501)], now=NOW, res=None, full=True
)
assert not fresh.stale
@@ -183,14 +193,14 @@ def test_alarm_not_repeated_within_a_day(tmp_path):
every=timedelta(hours=24),
alerted=NOW - timedelta(hours=5),
)
assert look(w, [], now=NOW, res=None).alarm is None
assert look(w, [], now=NOW, res=None, full=True).alarm is None
w.alerted = NOW - timedelta(hours=25)
assert look(w, [], now=NOW, res=None).alarm == "silence"
assert look(w, [], now=NOW, res=None, full=True).alarm == "silence"
def test_alarm_text_names_three_causes_and_suspects_the_channel(tmp_path):
w = state(tmp_path, seen=NOW - timedelta(hours=40), every=timedelta(hours=24))
report = look(w, [], now=NOW, res=timedelta(hours=24))
report = look(w, [], now=NOW, res=timedelta(hours=24), full=True)
silent = alarm_text(report, mine_sent=True)
assert "машина соседа" in silent and "крон" in silent and "канал" in silent
assert "канал подозревать надо раньше" in silent
@@ -208,6 +218,7 @@ def test_alarm_text_names_three_causes_and_suspects_the_channel(tmp_path):
],
now=NOW,
res=None,
full=True,
)
assert heard.heard
assert "канал скорее жив" in alarm_text(heard, mine_sent=True)
@@ -234,16 +245,17 @@ def test_state_survives_a_round_trip(tmp_path):
class FakeRoom:
def __init__(self, events, *, nick="beaver", send_fails=False):
def __init__(self, events, *, nick="beaver", send_fails=False, full=True):
self.events = events
self.nick = nick
self.send_fails = send_fails
self.full = full
self.said = []
self.limit = None
async def pending(self, limit=50):
async def window(self, limit=25):
self.limit = limit
return self.events
return Window(events=self.events, full=self.full)
async def say(self, text):
if self.send_fails:
@@ -310,7 +322,7 @@ async def test_watch_job_keeps_quiet_when_the_room_is_unreadable(tmp_path, monke
from beaver_agent.jobs import heartbeat as job
class Dead(FakeRoom):
async def pending(self, limit=50):
async def window(self, limit=25):
msg = "хаб не отвечает"
raise RuntimeError(msg)
@@ -320,3 +332,128 @@ async def test_watch_job_keeps_quiet_when_the_room_is_unreadable(tmp_path, monke
await job.watch(cast("Any", run))
assert not run.spawned
assert not (tmp_path / "сторож.json").exists()
class FakeHub:
"""Хаб как он есть: id сквозные по ВСЕМ комнатам, `/api/state` - наша.
Хвост комнаты лежит за `/api/state?view=…&after=<id>` и отдаётся
страницами вперёд; `/api/pending` - курсор, и сторожу в него ходить
нечего: то же событие второй раз оттуда не придёт.
"""
def __init__(self, events, *, page=PAGE):
self.events = events
self.page = page
self.said = []
self.afters = []
async def call(self, method, path, body):
if method == "POST" and path == "/api/messages":
self.said.append(body["body"])
return {"ok": True}
if path == "/api/room":
return {"room": {"id": "dbb", "name": "kai-rem-watch", "viewToken": "v-1"}}
if path.startswith("/api/state"):
query = dict(parse_qsl(urlsplit(path).query))
assert query["view"] == "v-1"
after = int(query.get("after", 0))
self.afters.append(after)
rest = [e for e in self.events if e["id"] > after]
return {"room": {}, "agents": [], "events": rest[: self.page]}
msg = f"сторож не должен ходить в {path}"
raise AssertionError(msg)
def raw(eid, nick, kind, body=None, at="2026-09-23T05:00:02.000Z"):
return {
"id": eid,
"roomId": "dbb",
"nick": nick,
"kind": kind,
"payload": {} if body is None else {"body": body},
"createdAt": at,
}
def room(monkeypatch, events, *, page=PAGE):
hub = FakeHub(events, page=page)
async def call(self, method, path, body):
return await hub.call(method, path, body)
monkeypatch.setattr(Vibegram, "_call", call)
return hub, Vibegram(hub="http://hub", token="t", nick="beaver-rem")
KAI_HB = (
f"hb {PEER} at=2026-09-23T05:00:01Z boot=2026-09-14T08:21:55Z "
"every=24h src=cron+uptime-s"
)
async def test_window_is_the_rooms_last_messages_not_the_hubs_last_ids(
tmp_path, monkeypatch, caplog
):
from beaver_agent.jobs import heartbeat as job
# В комнате три события, а по хабу за то же время прошло три десятка:
# номера сквозные, и между 296 и 336 лежат чужие комнаты. Отметка соседа
# (#329) обязана быть в окне - в комнате она первая с конца.
hub, client = room(
monkeypatch,
[
raw(296, PEER, "agent_join"),
raw(329, PEER, "message", KAI_HB),
raw(331, "beaver-rem", "agent_leave"),
raw(332, "beaver-rem", "message", "hb beaver-rem at=2026-09-23T15:12:01Z"),
raw(336, PEER, "message", "ack beaver-rem last=2026-09-23T15:12:01Z"),
],
)
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "сторож.json")
monkeypatch.setattr(job, "VIBEGRAM_HB", client)
window = await client.window(25)
assert window.full
assert [e.id for e in window.events] == [329, 332, 336] # join/leave не в счёт
assert parse_beat(window.events[0].text) is not None
# Два прогона подряд: отметка соседа в окне оба раза, а не только в первом
with caplog.at_level(logging.INFO, logger="beaver_agent.jobs.heartbeat"):
await job.watch(cast("Any", FakeRun()))
await job.watch(cast("Any", FakeRun()))
beats = [m for m in caplog.messages if "отметок соседа в окне: 1" in m]
assert len(beats) == 2
assert all("история дочитана" in m for m in beats)
assert all("last=2026-09-23T05:00:01Z" in a and "armed=yes" in a for a in hub.said)
assert all("win=25 " in a for a in hub.said)
assert hub.afters == [0, 0, 0] # одна страница на прогон, читать дальше нечего
async def test_a_hundred_noise_lines_do_not_push_the_control_beat_out(
tmp_path, monkeypatch
):
from beaver_agent.jobs import heartbeat as job
# Учения соседа: сотня шумовых строк в комнату и контрольная hb после них.
noise = [raw(400 + i, PEER, "message", f"шум {i}") for i in range(100)]
hub, client = room(monkeypatch, [*noise, raw(600, PEER, "message", KAI_HB)])
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "сторож.json")
monkeypatch.setattr(job, "VIBEGRAM_HB", client)
await job.watch(cast("Any", FakeRun()))
assert "last=2026-09-23T05:00:01Z" in hub.said[0] and "armed=yes" in hub.said[0]
assert Watch.load(tmp_path / "сторож.json").top == 600
def test_unread_window_is_named_and_takes_no_alarm(tmp_path):
late = {"seen": NOW - timedelta(hours=40), "every": timedelta(hours=24)}
partial = look(state(tmp_path, **late), [], now=NOW, res=None, full=False)
assert not partial.full and partial.alarm is None
assert "win=25/partial" in partial.ack
assert "окно не дочитано" in partial.note
# То же пустое окно, но дочитанное, - это показание, и оно даёт тревогу
whole = look(state(tmp_path, **late), [], now=NOW, res=None, full=True)
assert whole.alarm == "silence" and "win=25 " in whole.ack
assert "сообщений комнаты: 0 из 25, история дочитана" in whole.note
+64 -1
View File
@@ -1,10 +1,12 @@
from urllib.parse import parse_qsl, urlsplit
import pytest
from beaver_gateway.mcp.types import McpServer
from beaver_gateway.mcp.wrap import build_python_tool_server
from fastmcp import Client
from fastmcp.exceptions import ToolError
from beaver_agent.hands.vibegram import Vibegram, _cards, _event, _work
from beaver_agent.hands.vibegram import PAGE, Vibegram, _cards, _event, _work
def test_events_render_and_unknown_kinds_are_skipped():
@@ -64,3 +66,64 @@ async def test_tool_schema_lists_actions_and_needs_text_for_send():
)
with pytest.raises(ToolError, match="text"):
await client.call_tool("vibegram", {"action": "send"})
def _raw(eid, kind, body=None):
return {
"id": eid,
"roomId": "r",
"nick": "claude-kai",
"kind": kind,
"payload": {} if body is None else {"body": body},
"createdAt": "2026-09-23T05:00:02.000Z",
}
def _hub(monkeypatch, events):
calls = []
async def call(self, method, path, body):
calls.append(path)
if path == "/api/room":
return {"room": {"id": "r", "name": "kai-rem-watch", "viewToken": "v 1"}}
query = dict(parse_qsl(urlsplit(path).query))
assert query["view"] == "v 1" # ключ комнаты уезжает экранированным
after = int(query.get("after", 0))
return {"events": [e for e in events if e["id"] > after][:PAGE]}
monkeypatch.setattr(Vibegram, "_call", call)
return calls, Vibegram(hub="http://hub", token="t", nick="beaver-test")
async def test_window_pages_forward_to_the_end_and_drops_joins(monkeypatch):
# 520 событий комнаты - три страницы по 200; join/leave в окно не идут
events = [
_raw(i, "message" if i % 4 else "agent_leave", f"строка {i}")
for i in range(1, 521)
]
calls, room = _hub(monkeypatch, events)
window = await room.window(25)
assert window.full
assert len(window.events) == 25
assert window.events[-1].text.endswith("строка 519")
assert all("вошёл" not in e.text and "вышел" not in e.text for e in window.events)
assert calls.count("/api/room") == 1
assert sum(p.startswith("/api/state") for p in calls) == 3
assert "view=v%201" in calls[1]
assert not any("pending" in p for p in calls) # курсор сторожу не окно
async def test_window_that_ran_out_of_pages_says_it_is_not_full(monkeypatch):
events = [_raw(i, "message", f"строка {i}") for i in range(1, 521)]
_, room = _hub(monkeypatch, events)
assert not (await room.window(25, pages=2)).full
assert (await room.window(25, pages=3)).full
async def test_window_of_an_empty_room_is_full_and_empty(monkeypatch):
_, room = _hub(monkeypatch, [])
window = await room.window(25)
assert window.full and window.events == []