fix(pty,turn,backend): verify submit via jsonl, killpg orphans, idle session ttl

This commit is contained in:
hh
2026-07-26 00:24:40 +02:00
parent 4f6585b1ac
commit 89065c2f4e
4 changed files with 365 additions and 118 deletions
+2
View File
@@ -43,6 +43,7 @@ from claude_code_api.models import (
is_valid_model, is_valid_model,
) )
from claude_code_api.normalizer import normalize from claude_code_api.normalizer import normalize
from claude_code_api.pty import kill_orphaned_processes
from claude_code_api.turn import TurnManager from claude_code_api.turn import TurnManager
__version__ = "0.1.0" __version__ = "0.1.0"
@@ -76,6 +77,7 @@ __all__ = [
"UserMessage", "UserMessage",
"classify_pty_failure", "classify_pty_failure",
"is_valid_model", "is_valid_model",
"kill_orphaned_processes",
"normalize", "normalize",
"synthesize_turn_messages", "synthesize_turn_messages",
] ]
+117 -4
View File
@@ -22,6 +22,7 @@ import json
import logging import logging
import os import os
import tempfile import tempfile
import time
import uuid import uuid
from collections.abc import AsyncIterator, Callable, Iterable, Mapping from collections.abc import AsyncIterator, Callable, Iterable, Mapping
from dataclasses import dataclass, field from dataclasses import dataclass, field
@@ -55,6 +56,10 @@ HistoryInjectionMode = Literal["native_jsonl", "concat_message"]
ParseErrorCallback = Callable[[MessageParseError, dict[str, Any]], None] ParseErrorCallback = Callable[[MessageParseError, dict[str, Any]], None]
# How often the reaper wakes up to look for idle sessions. Far shorter
# than any sane TTL — the cost is one dict scan.
_REAP_INTERVAL = 60.0
@dataclass(frozen=True) @dataclass(frozen=True)
class BackendOptions: class BackendOptions:
@@ -88,6 +93,19 @@ class BackendOptions:
file_wait_timeout: float = 30.0 file_wait_timeout: float = 30.0
turn_duration_timeout: float = 5.0 turn_duration_timeout: float = 5.0
idle_session_ttl: float = 1800.0
"""Seconds a pooled session may sit unused before it is terminated.
The pool is keyed by history fingerprint, so a conversation that
forks (edited transcript, a reply that never made it back to the
client) leaves its previous session behind under a key nothing will
ever look up again. Without a TTL those sessions accumulate one live
``claude`` each — on a 4 GB host a handful is the whole machine.
There is deliberately no count cap: a burst of genuinely concurrent
conversations should all stay warm. Set to ``0`` to disable reaping.
"""
@dataclass @dataclass
class _LiveSession: class _LiveSession:
@@ -101,6 +119,25 @@ class _LiveSession:
def session_id(self) -> str: def session_id(self) -> str:
return self.pty.session_id return self.pty.session_id
@property
def idle_seconds(self) -> float:
"""Seconds since the last prompt, or 0 for a PTY that can't say.
Both accessors tolerate a PTY without the timestamps: the
``_session_factory`` seam exists so callers can inject minimal
fakes, and session bookkeeping must not widen that contract. A
fake reporting 0 is simply never idle enough to reap.
"""
last = getattr(self.pty, "last_activity_at", None)
if last is None:
return 0.0
return max(0.0, time.time() - last)
def touch(self) -> None:
touch = getattr(self.pty, "touch", None)
if touch is not None:
touch()
async def aclose(self) -> None: async def aclose(self) -> None:
await self.tm.aclose() await self.tm.aclose()
@@ -141,6 +178,7 @@ class ClaudeCodeBackend:
self._session_factory = _session_factory self._session_factory = _session_factory
self._closed = False self._closed = False
self._lock = asyncio.Lock() self._lock = asyncio.Lock()
self._reaper: asyncio.Task[None] | None = None
@property @property
def options(self) -> BackendOptions: def options(self) -> BackendOptions:
@@ -205,6 +243,7 @@ class ClaudeCodeBackend:
session: _LiveSession session: _LiveSession
if prior and fp_prior in self._sessions: if prior and fp_prior in self._sessions:
session = self._sessions.pop(fp_prior) session = self._sessions.pop(fp_prior)
session.touch()
send_text = last_text send_text = last_text
_log.info( _log.info(
"complete: POOL HIT fp=%s -> reusing session_id=%s", "complete: POOL HIT fp=%s -> reusing session_id=%s",
@@ -212,10 +251,22 @@ class ClaudeCodeBackend:
session.session_id, session.session_id,
) )
else: else:
# A miss means the caller's history no longer matches any
# session we're holding — almost always because the caller
# rewrote its own transcript, not because this is a new
# conversation. Log the pool's shape so that's diagnosable
# from the outside instead of by attaching a debugger.
_log.info( _log.info(
"complete: POOL MISS fp=%s (prior=%d msgs) -> spawning new session", "complete: POOL MISS fp=%s (prior=%d msgs) -> spawning new "
"session; pooled=%d [%s]",
fp_prior[:12], fp_prior[:12],
len(prior), len(prior),
len(self._sessions),
", ".join(
f"{fp[:12]}:{s.session_id[:8]}:idle{s.idle_seconds:.0f}s"
for fp, s in self._sessions.items()
)
or "empty",
) )
session = await self._create_session(prior) session = await self._create_session(prior)
if prior and self._opts.history_injection_mode == "concat_message": if prior and self._opts.history_injection_mode == "concat_message":
@@ -276,12 +327,70 @@ class ClaudeCodeBackend:
new_fp[:12], new_fp[:12],
) )
async def _reap_idle_sessions(self) -> None:
"""Terminate pooled sessions that have sat unused past the TTL.
Only ``_sessions`` is eligible — a session in ``_active`` is
mid-turn by definition. Runs under the same lock as ``complete``
so a session can't be reaped between fingerprint lookup and use.
"""
ttl = self._opts.idle_session_ttl
if ttl <= 0:
return
async with self._lock:
stale = [
(fp, s) for fp, s in self._sessions.items() if s.idle_seconds > ttl
]
for fp, _ in stale:
del self._sessions[fp]
for fp, s in stale:
_log.info(
"reaper: terminating session_id=%s (idle %.0fs > ttl %.0fs, fp=%s)",
s.session_id,
s.idle_seconds,
ttl,
fp[:12],
)
with contextlib.suppress(Exception):
await s.aclose()
if stale:
_log.info(
"reaper: reaped %d session(s), %d remain pooled",
len(stale),
len(self._sessions),
)
async def _reaper_loop(self) -> None:
while not self._closed:
await asyncio.sleep(_REAP_INTERVAL)
try:
await self._reap_idle_sessions()
except asyncio.CancelledError:
raise
except Exception:
_log.exception("reaper: pass failed — continuing")
async def aclose(self) -> None: async def aclose(self) -> None:
"""Shut down all live sessions; remove the temp mcp-config file.""" """Shut down every live session; remove the temp mcp-config file.
``_active`` is drained alongside ``_sessions``: a turn in flight
during shutdown used to leave its ``claude`` running forever,
because ptyprocess puts each child in its own session (setsid),
so it never sees the Ctrl-C that killed the gateway.
"""
self._closed = True self._closed = True
sessions = list(self._sessions.values()) reaper = self._reaper
self._reaper = None
if reaper is not None and not reaper.done():
reaper.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
await reaper
sessions = {id(s): s for s in self._sessions.values()}
sessions.update({id(s): s for s in self._active.values()})
self._sessions.clear() self._sessions.clear()
for s in sessions: self._active.clear()
_log.info("aclose: terminating %d live session(s)", len(sessions))
for s in sessions.values():
with contextlib.suppress(Exception): with contextlib.suppress(Exception):
await s.aclose() await s.aclose()
if self._mcp_config_path is not None: if self._mcp_config_path is not None:
@@ -290,6 +399,10 @@ class ClaudeCodeBackend:
self._mcp_config_path = None self._mcp_config_path = None
async def __aenter__(self) -> Self: async def __aenter__(self) -> Self:
if self._reaper is None and self._opts.idle_session_ttl > 0:
self._reaper = asyncio.create_task(
self._reaper_loop(), name="claude-session-reaper"
)
return self return self
async def __aexit__(self, _exc_type: object, _exc: object, _tb: object) -> None: async def __aexit__(self, _exc_type: object, _exc: object, _tb: object) -> None:
+145 -82
View File
@@ -13,6 +13,7 @@ This module knows nothing about turns, JSONL, or event normalization.
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import atexit
import contextlib import contextlib
import errno import errno
import logging import logging
@@ -21,6 +22,7 @@ import pathlib
import select import select
import signal import signal
import threading import threading
import time
import uuid import uuid
from collections.abc import Callable, Iterable, Mapping from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass, field from dataclasses import dataclass, field
@@ -34,6 +36,56 @@ _log = logging.getLogger("claude_code_api.pty")
PtyOutputCallback = Callable[[bytes], None] PtyOutputCallback = Callable[[bytes], None]
# ptyprocess spawns via ``pty.fork()``, which calls ``setsid()`` in the
# child — every claude lands in its own session and process group, with
# pgid == pid. Two consequences:
#
# * a Ctrl-C in the gateway's terminal goes to the *gateway's* foreground
# process group and never reaches claude, so orphans survive unless we
# kill them explicitly;
# * killing the group (rather than just the pid) also takes down whatever
# claude spawned — MCP stdio servers, ripgrep, node workers.
#
# We keep the pgids of every live process here so an ``atexit`` hook can
# sweep anything a disorderly shutdown left behind. Nothing saves us from
# SIGKILL on the gateway itself; everything short of that is covered.
_LIVE_PGIDS: set[int] = set()
_LIVE_PGIDS_LOCK = threading.Lock()
def _register_live(pgid: int) -> None:
with _LIVE_PGIDS_LOCK:
_LIVE_PGIDS.add(pgid)
def _unregister_live(pgid: int) -> None:
with _LIVE_PGIDS_LOCK:
_LIVE_PGIDS.discard(pgid)
def kill_orphaned_processes() -> int:
"""SIGKILL every process group we spawned and haven't reaped.
Registered as an ``atexit`` hook, and safe to call directly (e.g.
from a SIGTERM handler). Returns the number of groups signalled.
"""
with _LIVE_PGIDS_LOCK:
pgids = sorted(_LIVE_PGIDS)
_LIVE_PGIDS.clear()
killed = 0
for pgid in pgids:
try:
os.killpg(pgid, signal.SIGKILL)
except (OSError, ProcessLookupError):
continue
killed += 1
if killed:
_log.warning("kill_orphaned_processes: SIGKILLed %d orphan group(s)", killed)
return killed
atexit.register(kill_orphaned_processes)
_PROVIDER_ENV_VARS: tuple[str, ...] = ( _PROVIDER_ENV_VARS: tuple[str, ...] = (
"ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY",
"ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_AUTH_TOKEN",
@@ -64,21 +116,13 @@ _SNAPSHOT_INTERVAL = 10.0
# the paste handler is still scheduling when Enter arrives. # the paste handler is still scheduling when Enter arrives.
_SUBMIT_KEY_DELAY = 0.25 _SUBMIT_KEY_DELAY = 0.25
# After sending Enter we poll the PTY output for the paste-indicator # Submit verification does NOT live here — it lives in ``TurnManager``,
# (claude's TUI renders ``[Pasted text #...`` while the paste is still # which can see the session JSONL. The PTY layer has no trustworthy
# buffered in the input box). If it's still there after this window, # signal: claude's TUI renders ``[Pasted text #N +M lines]`` only for
# Enter never registered — we retry up to ``_SUBMIT_RETRIES`` times. # long/multi-line pastes, so a short prompt produces no indicator at all
_SUBMIT_VERIFY_WINDOW = 1.5 # and any "indicator is gone ⇒ submitted" check passes vacuously. The
_SUBMIT_VERIFY_POLL = 0.1 # JSONL user record is ground truth, so ``send_submit_key()`` is exposed
_SUBMIT_RETRIES = 3 # for the turn layer to re-press Enter when that record doesn't appear.
_PASTE_INDICATOR = b"[Pasted text"
# Size of the most-recent slice of PTY output considered "the current
# screen render" when verifying a submit. The TUI emits a full clear +
# redraw per state change; the latest redraw fits comfortably in this
# window, while earlier intermediate frames (which may still show the
# paste indicator) live further back in the same buffer.
_TUI_TAIL_WINDOW = 8192
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -236,11 +280,34 @@ class PtyClaudeProcess:
self._output_lock = threading.Lock() self._output_lock = threading.Lock()
self._output_buffer = bytearray() self._output_buffer = bytearray()
self._snapshot_task: asyncio.Task[None] | None = None self._snapshot_task: asyncio.Task[None] | None = None
self._created_at = time.time()
self._last_activity_at = self._created_at
@property @property
def session_id(self) -> str: def session_id(self) -> str:
return self._session_id return self._session_id
@property
def created_at(self) -> float:
"""Unix timestamp of construction (not of ``start()``)."""
return self._created_at
@property
def last_activity_at(self) -> float:
"""Unix timestamp of the last prompt written to this PTY.
Drives idle eviction in ``ClaudeCodeBackend`` and the
most-recent-first ordering in the admin PTY list. Deliberately
tracks *prompts*, not drain-thread output: a session rendering a
spinner is not being used, and claude redraws its TUI clock
forever, which would make every session look permanently busy.
"""
return self._last_activity_at
def touch(self) -> None:
"""Mark this session as used right now."""
self._last_activity_at = time.time()
@property @property
def argv(self) -> list[str]: def argv(self) -> list[str]:
return list(self._argv) return list(self._argv)
@@ -318,6 +385,7 @@ class PtyClaudeProcess:
self._session_id, self._session_id,
self._pty.pid, self._pty.pid,
) )
_register_live(self._pgid())
self._drain_stop.clear() self._drain_stop.clear()
self._drain_thread = threading.Thread( self._drain_thread = threading.Thread(
target=self._drain_loop, target=self._drain_loop,
@@ -366,35 +434,6 @@ class PtyClaudeProcess:
if overflow > 0: if overflow > 0:
del self._output_buffer[:overflow] del self._output_buffer[:overflow]
async def _wait_for_submit(self, pre_enter_len: int) -> bool:
"""Poll the PTY output to confirm Enter actually submitted the paste.
Claude's TUI renders ``[Pasted text #N +M lines]`` in the input
box while the paste is still buffered (unsubmitted). After Enter
registers, the input clears and the TUI redraws — the next
screen render no longer contains the indicator.
We compare bytes written *after* the Enter write (``pre_enter_len``
was snapshotted just before). Within those new bytes, only the
most recent ``_TUI_TAIL_WINDOW`` matter — that's the current
rendered screen; earlier bytes are intermediate redraws (one of
which would still show the indicator).
Returns True once the indicator is absent from the recent tail,
False on timeout.
"""
deadline = asyncio.get_running_loop().time() + _SUBMIT_VERIFY_WINDOW
while asyncio.get_running_loop().time() < deadline:
buf = self.captured_output()
new_bytes = buf[pre_enter_len:]
tail = new_bytes[-_TUI_TAIL_WINDOW:]
# Only check once the TUI has produced at least one redraw
# after our Enter (otherwise tail is empty / pre-render).
if tail and _PASTE_INDICATOR not in tail:
return True
await asyncio.sleep(_SUBMIT_VERIFY_POLL)
return False
async def _snapshot_loop(self, snapshot_dir: pathlib.Path) -> None: async def _snapshot_loop(self, snapshot_dir: pathlib.Path) -> None:
"""Periodically dump the captured PTY buffer to a file. """Periodically dump the captured PTY buffer to a file.
@@ -573,6 +612,11 @@ class PtyClaudeProcess:
3. A separate write sends the `\r` as a fresh keystroke that 3. A separate write sends the `\r` as a fresh keystroke that
triggers Submit. triggers Submit.
That Enter is best-effort: the TUI can still swallow it under
load, and this layer cannot tell. `TurnManager` confirms the
submit against the session JSONL and calls `send_submit_key()`
to re-press Enter when nothing shows up.
Callers that need raw byte streaming (e.g. arrow keys, individual Callers that need raw byte streaming (e.g. arrow keys, individual
keypresses) pass `newline=False` and write the framing themselves. keypresses) pass `newline=False` and write the framing themselves.
""" """
@@ -581,6 +625,7 @@ class PtyClaudeProcess:
raise RuntimeError(msg) raise RuntimeError(msg)
payload = data.encode("utf-8") if isinstance(data, str) else bytes(data) payload = data.encode("utf-8") if isinstance(data, str) else bytes(data)
pty = self._pty pty = self._pty
self.touch()
if not newline: if not newline:
_log.info( _log.info(
"write: session_id=%s RAW %d bytes (newline=False)", "write: session_id=%s RAW %d bytes (newline=False)",
@@ -599,45 +644,31 @@ class PtyClaudeProcess:
) )
n1 = await asyncio.to_thread(pty.write, paste_chunk) n1 = await asyncio.to_thread(pty.write, paste_chunk)
await asyncio.sleep(_SUBMIT_KEY_DELAY) await asyncio.sleep(_SUBMIT_KEY_DELAY)
n2 = 0 n2 = await asyncio.to_thread(pty.write, b"\r")
for attempt in range(1, _SUBMIT_RETRIES + 1):
# Snapshot just before each Enter write so the verifier only
# looks at bytes the TUI produces in response to *this*
# Enter, not earlier renders (which still show the paste
# indicator from when the paste was first buffered).
pre_enter_len = len(self.captured_output())
n2 += await asyncio.to_thread(pty.write, b"\r")
submitted = await self._wait_for_submit(pre_enter_len)
if submitted:
if attempt > 1:
_log.info( _log.info(
"write: session_id=%s SUBMIT registered after %d Enter(s)", "write: session_id=%s paste=%d bytes + Enter=%d bytes written "
self._session_id, "(submit unconfirmed until JSONL moves)",
attempt,
)
break
_log.warning(
"write: session_id=%s Enter #%d did not clear paste indicator "
"(buf=%d bytes) — retrying",
self._session_id,
attempt,
len(self.captured_output()),
)
else:
_log.error(
"write: session_id=%s SUBMIT never registered after %d "
"Enter attempts — paste likely stuck in input box",
self._session_id,
_SUBMIT_RETRIES,
)
_log.info(
"write: session_id=%s SUBMIT done (paste=%d Enter=%d)",
self._session_id, self._session_id,
n1, n1,
n2, n2,
) )
return n1 + n2 return n1 + n2
async def send_submit_key(self) -> int:
r"""Press Enter again, on its own, to submit a stuck paste.
Used by `TurnManager` when the JSONL shows no sign that the
previous Enter registered. Writing a bare `\r` is safe to repeat:
if the input box is already empty, claude treats it as a no-op.
"""
if self._pty is None:
msg = "PtyClaudeProcess not started"
raise RuntimeError(msg)
_log.warning(
"send_submit_key: session_id=%s re-pressing Enter", self._session_id
)
return await asyncio.to_thread(self._pty.write, b"\r")
async def send_control(self, char: str) -> None: async def send_control(self, char: str) -> None:
"""Send a control character (e.g. 'c' for Ctrl-C, 'd' for Ctrl-D).""" """Send a control character (e.g. 'c' for Ctrl-C, 'd' for Ctrl-D)."""
if self._pty is None: if self._pty is None:
@@ -653,20 +684,50 @@ class PtyClaudeProcess:
pty = self._pty pty = self._pty
return await asyncio.to_thread(pty.wait) return await asyncio.to_thread(pty.wait)
def _pgid(self) -> int:
"""Process-group id of the child.
``pty.fork()`` makes the child a session leader, so its pgid
equals its pid; we still ask the kernel rather than assume it.
"""
pty = self._pty
if pty is None:
msg = "PtyClaudeProcess not started"
raise RuntimeError(msg)
try:
return os.getpgid(pty.pid)
except OSError:
return pty.pid
def _signal_group(self, sig: int) -> None:
"""Signal claude *and everything it spawned*.
Signalling only the pid leaves claude's own children behind —
MCP stdio servers, node workers, long-running Bash tools — which
is how a "terminated" session keeps holding hundreds of MB.
Falls back to a plain pid kill if the group is already gone.
"""
pty = self._pty
if pty is None:
return
try:
os.killpg(self._pgid(), sig)
except (OSError, ProcessLookupError):
with contextlib.suppress(OSError):
pty.kill(sig)
async def terminate(self, *, grace: float = 5.0) -> int | None: async def terminate(self, *, grace: float = 5.0) -> int | None:
"""SIGTERM → wait up to `grace` seconds → SIGKILL ladder.""" """SIGTERM → wait up to `grace` seconds → SIGKILL ladder."""
if self._pty is None: if self._pty is None:
return None return None
pty = self._pty pty = self._pty
if pty.isalive(): if pty.isalive():
with contextlib.suppress(OSError): self._signal_group(signal.SIGTERM)
pty.kill(signal.SIGTERM)
deadline = asyncio.get_running_loop().time() + grace deadline = asyncio.get_running_loop().time() + grace
while pty.isalive() and asyncio.get_running_loop().time() < deadline: while pty.isalive() and asyncio.get_running_loop().time() < deadline:
await asyncio.sleep(0.05) await asyncio.sleep(0.05)
if pty.isalive(): if pty.isalive():
with contextlib.suppress(OSError): self._signal_group(signal.SIGKILL)
pty.kill(signal.SIGKILL)
return await self._reap() return await self._reap()
async def kill(self) -> int | None: async def kill(self) -> int | None:
@@ -675,14 +736,15 @@ class PtyClaudeProcess:
return None return None
pty = self._pty pty = self._pty
if pty.isalive(): if pty.isalive():
with contextlib.suppress(OSError): self._signal_group(signal.SIGKILL)
pty.kill(signal.SIGKILL)
return await self._reap() return await self._reap()
async def _reap(self) -> int | None: async def _reap(self) -> int | None:
pty = self._pty pty = self._pty
if pty is None: if pty is None:
return None return None
with contextlib.suppress(RuntimeError):
_unregister_live(self._pgid())
exit_status = await asyncio.to_thread(pty.wait) exit_status = await asyncio.to_thread(pty.wait)
self._drain_stop.set() self._drain_stop.set()
thread = self._drain_thread thread = self._drain_thread
@@ -720,4 +782,5 @@ __all__: Iterable[str] = (
"PtyProcessOptions", "PtyProcessOptions",
"build_argv", "build_argv",
"build_env", "build_env",
"kill_orphaned_processes",
) )
+97 -28
View File
@@ -64,6 +64,20 @@ _DEFAULT_STARTUP_DELAY = 10.0
_TUI_READY_MARKER: bytes = b"\x1b[?2004h" _TUI_READY_MARKER: bytes = b"\x1b[?2004h"
_TUI_QUIET_PERIOD: float = 1.5 _TUI_QUIET_PERIOD: float = 1.5
# Submitting a prompt is not reliably observable from the PTY: claude's
# TUI only renders a ``[Pasted text #N]`` placeholder for long/multi-line
# pastes, so short prompts leave no trace to check against and the Enter
# that follows a paste can be swallowed by the Ink stdin reader under
# load. The session JSONL *is* ground truth — claude appends a record as
# soon as a prompt is accepted. So we wait for the file to move, and
# re-press Enter if it doesn't.
#
# The window has to clear claude's own start-of-turn latency on a loaded
# Pi (spinner up, context assembled) without stranding the user behind a
# genuinely stuck input box for a minute.
_SUBMIT_CONFIRM_WINDOW: float = 12.0
_SUBMIT_RETRIES: int = 3
ParseErrorCallback = Callable[[MessageParseError, JsonlRecord], None] ParseErrorCallback = Callable[[MessageParseError, JsonlRecord], None]
@@ -257,37 +271,13 @@ class TurnManager:
fallback_message="claude process not accepting input" fallback_message="claude process not accepting input"
) from exc ) from exc
if not self._watcher.path.exists():
_log.info( _log.info(
"send_user_message: session_id=%s waiting for JSONL %s (timeout=%.1fs)", "send_user_message: session_id=%s JSONL %s %s",
sid,
self._watcher.path,
self._file_wait_timeout or 0.0,
)
try:
await self._watcher.wait_for_file(timeout=self._file_wait_timeout)
_log.info(
"send_user_message: session_id=%s JSONL appeared", sid
)
except TimeoutError as exc:
_log.error(
"send_user_message: session_id=%s JSONL DID NOT APPEAR within %.1fs at %s",
sid,
self._file_wait_timeout or 0.0,
self._watcher.path,
)
raise self._classify_pty_failure(
fallback_cls=SessionError,
fallback_message=(
f"JSONL file did not appear within "
f"{self._file_wait_timeout}s: {self._watcher.path}"
),
) from exc
else:
_log.info(
"send_user_message: session_id=%s JSONL already exists at %s (resume mode)",
sid, sid,
self._watcher.path, self._watcher.path,
"exists (resume/reuse)"
if self._watcher.path.exists()
else "not created yet (fresh session)",
) )
terminal_assistant: AssistantMessage | None = None terminal_assistant: AssistantMessage | None = None
@@ -300,10 +290,81 @@ class TurnManager:
n_events_yielded = 0 n_events_yielded = 0
last_progress_log = t_loop_start last_progress_log = t_loop_start
# Until the first record of this turn lands we cannot tell
# "claude is thinking" from "the Enter never registered", so
# we re-press Enter on a timer instead of hanging forever.
submit_confirmed = False
submit_attempts = 1
submit_deadline = t_loop_start + _SUBMIT_CONFIRM_WINDOW
while True: while True:
records = await self._watcher.read_once() records = await self._watcher.read_once()
if records and not submit_confirmed:
submit_confirmed = True
if submit_attempts > 1:
_log.info(
"send_user_message: session_id=%s SUBMIT confirmed "
"after %d Enter(s), %.1fs",
sid,
submit_attempts,
loop.time() - t_loop_start,
)
if not records: if not records:
now = loop.time() now = loop.time()
if (
not submit_confirmed
and self._file_wait_timeout is not None
and now - t_loop_start > self._file_wait_timeout
and not self._watcher.path.exists()
):
# The file never even got created. That is not a
# swallowed Enter — it is claude failing to open a
# session at all (auth / config), and no number of
# extra Enters will fix it.
_log.error(
"send_user_message: session_id=%s JSONL DID NOT APPEAR "
"within %.1fs at %s",
sid,
self._file_wait_timeout,
self._watcher.path,
)
raise self._classify_pty_failure(
fallback_cls=SessionError,
fallback_message=(
f"JSONL file did not appear within "
f"{self._file_wait_timeout}s: {self._watcher.path}"
),
)
if not submit_confirmed and now >= submit_deadline:
if submit_attempts > _SUBMIT_RETRIES:
_log.error(
"send_user_message: session_id=%s prompt NEVER "
"submitted — %d Enter(s) over %.1fs left the JSONL "
"at %s untouched",
sid,
submit_attempts,
now - t_loop_start,
self._watcher.path,
)
raise self._classify_pty_failure(
fallback_cls=SessionError,
fallback_message=(
f"prompt was never submitted: no JSONL record "
f"after {submit_attempts} Enter attempts over "
f"{now - t_loop_start:.0f}s ({self._watcher.path})"
),
)
_log.warning(
"send_user_message: session_id=%s no JSONL record %.1fs "
"after Enter #%d — paste likely stuck in the input box, "
"re-pressing Enter",
sid,
_SUBMIT_CONFIRM_WINDOW,
submit_attempts,
)
await self._send_submit_key()
submit_attempts += 1
submit_deadline = now + _SUBMIT_CONFIRM_WINDOW
# Heartbeat log every 10s so a hung wait is visible. # Heartbeat log every 10s so a hung wait is visible.
if now - last_progress_log >= 10.0: if now - last_progress_log >= 10.0:
_log.info( _log.info(
@@ -416,6 +477,14 @@ class TurnManager:
finally: finally:
self._turn_in_progress = False self._turn_in_progress = False
async def _send_submit_key(self) -> None:
"""Re-press Enter, tolerating PTY fakes that lack the method."""
send = getattr(self._pty, "send_submit_key", None)
if send is None:
return
with contextlib.suppress(OSError):
await send()
def _pty_is_alive(self) -> bool: def _pty_is_alive(self) -> bool:
"""Best-effort liveness check. Test fakes may lack `is_alive()`.""" """Best-effort liveness check. Test fakes may lack `is_alive()`."""
is_alive = getattr(self._pty, "is_alive", None) is_alive = getattr(self._pty, "is_alive", None)