fix(hands,jobs): watchdog window reads the room tail, not the pending cursor
This commit is contained in:
1 parent
91bd50a66d
commit
47c31ac356
6 files changed
+341
-37
No files matched your search
+158
-21
@@ -1,6 +1,8 @@
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from beaver_agent.hands.heartbeat import (
|
||||
@@ -14,7 +16,7 @@ from beaver_agent.hands.heartbeat import (
|
||||
parse_span,
|
||||
span,
|
||||
)
|
||||
from beaver_agent.hands.vibegram import Event
|
||||
from beaver_agent.hands.vibegram import PAGE, Event, Vibegram, Window
|
||||
|
||||
WARSAW = ZoneInfo("Europe/Warsaw")
|
||||
NOW = datetime(2026, 9, 22, 15, 42, tzinfo=UTC)
|
||||
@@ -100,7 +102,9 @@ def test_beat_line_says_unknown_instead_of_guessing():
|
||||
|
||||
def test_fresh_beat_arms_and_stays_quiet(tmp_path):
|
||||
w = state(tmp_path)
|
||||
report = look(w, [hb(NOW - timedelta(hours=10))], now=NOW, res=timedelta(hours=24))
|
||||
report = look(
|
||||
w, [hb(NOW - timedelta(hours=10))], now=NOW, res=timedelta(hours=24), full=True
|
||||
)
|
||||
assert report.armed and report.alarm is None and not report.stale
|
||||
assert report.threshold == timedelta(hours=36)
|
||||
assert "armed=yes" in report.ack and "win=25" in report.ack
|
||||
@@ -111,19 +115,21 @@ def test_fresh_beat_arms_and_stays_quiet(tmp_path):
|
||||
def test_threshold_is_every_times_one_and_a_half(tmp_path):
|
||||
# 30 часов при every=24h - ещё не тревога: одна потерянная строка не в счёт
|
||||
quiet = state(tmp_path, seen=NOW - timedelta(hours=30), every=timedelta(hours=24))
|
||||
assert look(quiet, [], now=NOW, res=timedelta(hours=24)).alarm is None
|
||||
assert look(quiet, [], now=NOW, res=timedelta(hours=24), full=True).alarm is None
|
||||
# 37 часов - уже за порогом 36
|
||||
loud = state(tmp_path, seen=NOW - timedelta(hours=37), every=timedelta(hours=24))
|
||||
report = look(loud, [], now=NOW, res=timedelta(hours=24))
|
||||
report = look(loud, [], now=NOW, res=timedelta(hours=24), full=True)
|
||||
assert report.alarm == "silence"
|
||||
# Порог берётся из строки соседа, а не из нашей константы
|
||||
slow = state(tmp_path, seen=NOW - timedelta(hours=37), every=timedelta(hours=48))
|
||||
assert look(slow, [], now=NOW, res=timedelta(hours=24)).alarm is None
|
||||
assert look(slow, [], now=NOW, res=timedelta(hours=24), full=True).alarm is None
|
||||
|
||||
|
||||
def test_last_beat_wins_over_gaps(tmp_path):
|
||||
w = state(tmp_path, seen=NOW - timedelta(hours=40), every=timedelta(hours=24))
|
||||
report = look(w, [hb(NOW - timedelta(hours=1), eid=120)], now=NOW, res=None)
|
||||
report = look(
|
||||
w, [hb(NOW - timedelta(hours=1), eid=120)], now=NOW, res=None, full=True
|
||||
)
|
||||
assert report.alarm is None
|
||||
assert report.age is not None and report.age < timedelta(hours=2)
|
||||
assert "res=unknown" in report.ack
|
||||
@@ -131,7 +137,7 @@ def test_last_beat_wins_over_gaps(tmp_path):
|
||||
|
||||
def test_unknown_every_falls_back_to_last_known_and_says_so(tmp_path):
|
||||
w = state(tmp_path, every=timedelta(hours=24))
|
||||
report = look(w, [hb(NOW, every="unknown", eid=130)], now=NOW, res=None)
|
||||
report = look(w, [hb(NOW, every="unknown", eid=130)], now=NOW, res=None, full=True)
|
||||
assert report.threshold == timedelta(hours=36)
|
||||
assert "thr=36h-fallback" in report.ack
|
||||
assert w.every == timedelta(hours=24)
|
||||
@@ -139,7 +145,7 @@ def test_unknown_every_falls_back_to_last_known_and_says_so(tmp_path):
|
||||
|
||||
def test_no_period_ever_means_no_measurement_and_no_alarm(tmp_path):
|
||||
w = state(tmp_path, seen=NOW - timedelta(days=9))
|
||||
report = look(w, [], now=NOW, res=timedelta(hours=24))
|
||||
report = look(w, [], now=NOW, res=timedelta(hours=24), full=True)
|
||||
assert report.alarm is None and report.threshold is None
|
||||
assert "thr=unknown" in report.ack
|
||||
assert "мерить нечем" in report.note
|
||||
@@ -147,7 +153,7 @@ def test_no_period_ever_means_no_measurement_and_no_alarm(tmp_path):
|
||||
|
||||
def test_never_seen_is_armed_no_not_alarm(tmp_path):
|
||||
w = state(tmp_path)
|
||||
report = look(w, [], now=NOW, res=timedelta(hours=24))
|
||||
report = look(w, [], now=NOW, res=timedelta(hours=24), full=True)
|
||||
assert not report.armed and report.alarm is None
|
||||
assert "armed=no" in report.ack and "last=never seen=never" in report.ack
|
||||
assert w.unarmed_since == NOW
|
||||
@@ -155,11 +161,11 @@ def test_never_seen_is_armed_no_not_alarm(tmp_path):
|
||||
|
||||
def test_seven_days_unarmed_tells_the_human_once(tmp_path):
|
||||
w = state(tmp_path, unarmed_since=NOW - timedelta(days=7))
|
||||
assert look(w, [], now=NOW, res=timedelta(hours=24)).alarm == "unarmed"
|
||||
assert look(w, [], now=NOW, res=timedelta(hours=24), full=True).alarm == "unarmed"
|
||||
w.unarmed_told = True
|
||||
assert look(w, [], now=NOW, res=timedelta(hours=24)).alarm is None
|
||||
assert look(w, [], now=NOW, res=timedelta(hours=24), full=True).alarm is None
|
||||
# Пришла первая отметка - кольцо вооружилось, счётчик забыт
|
||||
look(w, [hb(NOW, eid=140)], now=NOW, res=timedelta(hours=24))
|
||||
look(w, [hb(NOW, eid=140)], now=NOW, res=timedelta(hours=24), full=True)
|
||||
assert w.unarmed_since is None
|
||||
|
||||
|
||||
@@ -167,12 +173,16 @@ def test_stale_window_forbids_alarm(tmp_path):
|
||||
w = state(
|
||||
tmp_path, seen=NOW - timedelta(hours=40), every=timedelta(hours=24), top=500
|
||||
)
|
||||
report = look(w, [hb(NOW - timedelta(hours=40), eid=7)], now=NOW, res=None)
|
||||
report = look(
|
||||
w, [hb(NOW - timedelta(hours=40), eid=7)], now=NOW, res=None, full=True
|
||||
)
|
||||
assert report.stale and report.alarm is None
|
||||
assert "win=stale" in report.ack
|
||||
assert w.top == 500 # съехавшее окно не двигает отметку максимума
|
||||
# Окно снова вперёд - тревога берётся
|
||||
fresh = look(w, [hb(NOW - timedelta(hours=40), eid=501)], now=NOW, res=None)
|
||||
fresh = look(
|
||||
w, [hb(NOW - timedelta(hours=40), eid=501)], now=NOW, res=None, full=True
|
||||
)
|
||||
assert not fresh.stale
|
||||
|
||||
|
||||
@@ -183,14 +193,14 @@ def test_alarm_not_repeated_within_a_day(tmp_path):
|
||||
every=timedelta(hours=24),
|
||||
alerted=NOW - timedelta(hours=5),
|
||||
)
|
||||
assert look(w, [], now=NOW, res=None).alarm is None
|
||||
assert look(w, [], now=NOW, res=None, full=True).alarm is None
|
||||
w.alerted = NOW - timedelta(hours=25)
|
||||
assert look(w, [], now=NOW, res=None).alarm == "silence"
|
||||
assert look(w, [], now=NOW, res=None, full=True).alarm == "silence"
|
||||
|
||||
|
||||
def test_alarm_text_names_three_causes_and_suspects_the_channel(tmp_path):
|
||||
w = state(tmp_path, seen=NOW - timedelta(hours=40), every=timedelta(hours=24))
|
||||
report = look(w, [], now=NOW, res=timedelta(hours=24))
|
||||
report = look(w, [], now=NOW, res=timedelta(hours=24), full=True)
|
||||
silent = alarm_text(report, mine_sent=True)
|
||||
assert "машина соседа" in silent and "крон" in silent and "канал" in silent
|
||||
assert "канал подозревать надо раньше" in silent
|
||||
@@ -208,6 +218,7 @@ def test_alarm_text_names_three_causes_and_suspects_the_channel(tmp_path):
|
||||
],
|
||||
now=NOW,
|
||||
res=None,
|
||||
full=True,
|
||||
)
|
||||
assert heard.heard
|
||||
assert "канал скорее жив" in alarm_text(heard, mine_sent=True)
|
||||
@@ -234,16 +245,17 @@ def test_state_survives_a_round_trip(tmp_path):
|
||||
|
||||
|
||||
class FakeRoom:
|
||||
def __init__(self, events, *, nick="beaver", send_fails=False):
|
||||
def __init__(self, events, *, nick="beaver", send_fails=False, full=True):
|
||||
self.events = events
|
||||
self.nick = nick
|
||||
self.send_fails = send_fails
|
||||
self.full = full
|
||||
self.said = []
|
||||
self.limit = None
|
||||
|
||||
async def pending(self, limit=50):
|
||||
async def window(self, limit=25):
|
||||
self.limit = limit
|
||||
return self.events
|
||||
return Window(events=self.events, full=self.full)
|
||||
|
||||
async def say(self, text):
|
||||
if self.send_fails:
|
||||
@@ -310,7 +322,7 @@ async def test_watch_job_keeps_quiet_when_the_room_is_unreadable(tmp_path, monke
|
||||
from beaver_agent.jobs import heartbeat as job
|
||||
|
||||
class Dead(FakeRoom):
|
||||
async def pending(self, limit=50):
|
||||
async def window(self, limit=25):
|
||||
msg = "хаб не отвечает"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
@@ -320,3 +332,128 @@ async def test_watch_job_keeps_quiet_when_the_room_is_unreadable(tmp_path, monke
|
||||
await job.watch(cast("Any", run))
|
||||
assert not run.spawned
|
||||
assert not (tmp_path / "сторож.json").exists()
|
||||
|
||||
|
||||
class FakeHub:
|
||||
"""Хаб как он есть: id сквозные по ВСЕМ комнатам, `/api/state` - наша.
|
||||
|
||||
Хвост комнаты лежит за `/api/state?view=…&after=<id>` и отдаётся
|
||||
страницами вперёд; `/api/pending` - курсор, и сторожу в него ходить
|
||||
нечего: то же событие второй раз оттуда не придёт.
|
||||
"""
|
||||
|
||||
def __init__(self, events, *, page=PAGE):
|
||||
self.events = events
|
||||
self.page = page
|
||||
self.said = []
|
||||
self.afters = []
|
||||
|
||||
async def call(self, method, path, body):
|
||||
if method == "POST" and path == "/api/messages":
|
||||
self.said.append(body["body"])
|
||||
return {"ok": True}
|
||||
if path == "/api/room":
|
||||
return {"room": {"id": "dbb", "name": "kai-rem-watch", "viewToken": "v-1"}}
|
||||
if path.startswith("/api/state"):
|
||||
query = dict(parse_qsl(urlsplit(path).query))
|
||||
assert query["view"] == "v-1"
|
||||
after = int(query.get("after", 0))
|
||||
self.afters.append(after)
|
||||
rest = [e for e in self.events if e["id"] > after]
|
||||
return {"room": {}, "agents": [], "events": rest[: self.page]}
|
||||
msg = f"сторож не должен ходить в {path}"
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
def raw(eid, nick, kind, body=None, at="2026-09-23T05:00:02.000Z"):
|
||||
return {
|
||||
"id": eid,
|
||||
"roomId": "dbb",
|
||||
"nick": nick,
|
||||
"kind": kind,
|
||||
"payload": {} if body is None else {"body": body},
|
||||
"createdAt": at,
|
||||
}
|
||||
|
||||
|
||||
def room(monkeypatch, events, *, page=PAGE):
|
||||
hub = FakeHub(events, page=page)
|
||||
|
||||
async def call(self, method, path, body):
|
||||
return await hub.call(method, path, body)
|
||||
|
||||
monkeypatch.setattr(Vibegram, "_call", call)
|
||||
return hub, Vibegram(hub="http://hub", token="t", nick="beaver-rem")
|
||||
|
||||
|
||||
KAI_HB = (
|
||||
f"hb {PEER} at=2026-09-23T05:00:01Z boot=2026-09-14T08:21:55Z "
|
||||
"every=24h src=cron+uptime-s"
|
||||
)
|
||||
|
||||
|
||||
async def test_window_is_the_rooms_last_messages_not_the_hubs_last_ids(
|
||||
tmp_path, monkeypatch, caplog
|
||||
):
|
||||
from beaver_agent.jobs import heartbeat as job
|
||||
|
||||
# В комнате три события, а по хабу за то же время прошло три десятка:
|
||||
# номера сквозные, и между 296 и 336 лежат чужие комнаты. Отметка соседа
|
||||
# (#329) обязана быть в окне - в комнате она первая с конца.
|
||||
hub, client = room(
|
||||
monkeypatch,
|
||||
[
|
||||
raw(296, PEER, "agent_join"),
|
||||
raw(329, PEER, "message", KAI_HB),
|
||||
raw(331, "beaver-rem", "agent_leave"),
|
||||
raw(332, "beaver-rem", "message", "hb beaver-rem at=2026-09-23T15:12:01Z"),
|
||||
raw(336, PEER, "message", "ack beaver-rem last=2026-09-23T15:12:01Z"),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "сторож.json")
|
||||
monkeypatch.setattr(job, "VIBEGRAM_HB", client)
|
||||
|
||||
window = await client.window(25)
|
||||
assert window.full
|
||||
assert [e.id for e in window.events] == [329, 332, 336] # join/leave не в счёт
|
||||
assert parse_beat(window.events[0].text) is not None
|
||||
|
||||
# Два прогона подряд: отметка соседа в окне оба раза, а не только в первом
|
||||
with caplog.at_level(logging.INFO, logger="beaver_agent.jobs.heartbeat"):
|
||||
await job.watch(cast("Any", FakeRun()))
|
||||
await job.watch(cast("Any", FakeRun()))
|
||||
beats = [m for m in caplog.messages if "отметок соседа в окне: 1" in m]
|
||||
assert len(beats) == 2
|
||||
assert all("история дочитана" in m for m in beats)
|
||||
assert all("last=2026-09-23T05:00:01Z" in a and "armed=yes" in a for a in hub.said)
|
||||
assert all("win=25 " in a for a in hub.said)
|
||||
assert hub.afters == [0, 0, 0] # одна страница на прогон, читать дальше нечего
|
||||
|
||||
|
||||
async def test_a_hundred_noise_lines_do_not_push_the_control_beat_out(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
from beaver_agent.jobs import heartbeat as job
|
||||
|
||||
# Учения соседа: сотня шумовых строк в комнату и контрольная hb после них.
|
||||
noise = [raw(400 + i, PEER, "message", f"шум {i}") for i in range(100)]
|
||||
hub, client = room(monkeypatch, [*noise, raw(600, PEER, "message", KAI_HB)])
|
||||
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "сторож.json")
|
||||
monkeypatch.setattr(job, "VIBEGRAM_HB", client)
|
||||
|
||||
await job.watch(cast("Any", FakeRun()))
|
||||
|
||||
assert "last=2026-09-23T05:00:01Z" in hub.said[0] and "armed=yes" in hub.said[0]
|
||||
assert Watch.load(tmp_path / "сторож.json").top == 600
|
||||
|
||||
|
||||
def test_unread_window_is_named_and_takes_no_alarm(tmp_path):
|
||||
late = {"seen": NOW - timedelta(hours=40), "every": timedelta(hours=24)}
|
||||
partial = look(state(tmp_path, **late), [], now=NOW, res=None, full=False)
|
||||
assert not partial.full and partial.alarm is None
|
||||
assert "win=25/partial" in partial.ack
|
||||
assert "окно не дочитано" in partial.note
|
||||
# То же пустое окно, но дочитанное, - это показание, и оно даёт тревогу
|
||||
whole = look(state(tmp_path, **late), [], now=NOW, res=None, full=True)
|
||||
assert whole.alarm == "silence" and "win=25 " in whole.ack
|
||||
assert "сообщений комнаты: 0 из 25, история дочитана" in whole.note
|
||||
+64
-1
@@ -1,10 +1,12 @@
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
import pytest
|
||||
from beaver_gateway.mcp.types import McpServer
|
||||
from beaver_gateway.mcp.wrap import build_python_tool_server
|
||||
from fastmcp import Client
|
||||
from fastmcp.exceptions import ToolError
|
||||
|
||||
from beaver_agent.hands.vibegram import Vibegram, _cards, _event, _work
|
||||
from beaver_agent.hands.vibegram import PAGE, Vibegram, _cards, _event, _work
|
||||
|
||||
|
||||
def test_events_render_and_unknown_kinds_are_skipped():
|
||||
@@ -64,3 +66,64 @@ async def test_tool_schema_lists_actions_and_needs_text_for_send():
|
||||
)
|
||||
with pytest.raises(ToolError, match="text"):
|
||||
await client.call_tool("vibegram", {"action": "send"})
|
||||
|
||||
|
||||
def _raw(eid, kind, body=None):
|
||||
return {
|
||||
"id": eid,
|
||||
"roomId": "r",
|
||||
"nick": "claude-kai",
|
||||
"kind": kind,
|
||||
"payload": {} if body is None else {"body": body},
|
||||
"createdAt": "2026-09-23T05:00:02.000Z",
|
||||
}
|
||||
|
||||
|
||||
def _hub(monkeypatch, events):
|
||||
calls = []
|
||||
|
||||
async def call(self, method, path, body):
|
||||
calls.append(path)
|
||||
if path == "/api/room":
|
||||
return {"room": {"id": "r", "name": "kai-rem-watch", "viewToken": "v 1"}}
|
||||
query = dict(parse_qsl(urlsplit(path).query))
|
||||
assert query["view"] == "v 1" # ключ комнаты уезжает экранированным
|
||||
after = int(query.get("after", 0))
|
||||
return {"events": [e for e in events if e["id"] > after][:PAGE]}
|
||||
|
||||
monkeypatch.setattr(Vibegram, "_call", call)
|
||||
return calls, Vibegram(hub="http://hub", token="t", nick="beaver-test")
|
||||
|
||||
|
||||
async def test_window_pages_forward_to_the_end_and_drops_joins(monkeypatch):
|
||||
# 520 событий комнаты - три страницы по 200; join/leave в окно не идут
|
||||
events = [
|
||||
_raw(i, "message" if i % 4 else "agent_leave", f"строка {i}")
|
||||
for i in range(1, 521)
|
||||
]
|
||||
calls, room = _hub(monkeypatch, events)
|
||||
|
||||
window = await room.window(25)
|
||||
|
||||
assert window.full
|
||||
assert len(window.events) == 25
|
||||
assert window.events[-1].text.endswith("строка 519")
|
||||
assert all("вошёл" not in e.text and "вышел" not in e.text for e in window.events)
|
||||
assert calls.count("/api/room") == 1
|
||||
assert sum(p.startswith("/api/state") for p in calls) == 3
|
||||
assert "view=v%201" in calls[1]
|
||||
assert not any("pending" in p for p in calls) # курсор сторожу не окно
|
||||
|
||||
|
||||
async def test_window_that_ran_out_of_pages_says_it_is_not_full(monkeypatch):
|
||||
events = [_raw(i, "message", f"строка {i}") for i in range(1, 521)]
|
||||
_, room = _hub(monkeypatch, events)
|
||||
|
||||
assert not (await room.window(25, pages=2)).full
|
||||
assert (await room.window(25, pages=3)).full
|
||||
|
||||
|
||||
async def test_window_of_an_empty_room_is_full_and_empty(monkeypatch):
|
||||
_, room = _hub(monkeypatch, [])
|
||||
window = await room.window(25)
|
||||
assert window.full and window.events == []
|
||||
Reference in new issue
Block a user