403 lines
15 KiB
Python
403 lines
15 KiB
Python
"""SQLModel tables.
|
|
|
|
Flat tables, no FK relationships modelled (``actor`` and
|
|
``agent_name`` are stored as strings — joining audit→token by name is
|
|
fine at this volume; we'll introduce FKs when the admin UI actually
|
|
demands them).
|
|
|
|
The ``Conversation`` + ``ConversationMessage`` pair persists chat
|
|
history per frontend so we can survive cache misses without losing
|
|
tool-call memory. The gateway is now stateful about conversation
|
|
content (we keep the raw Anthropic-shape message list including
|
|
``tool_use`` / ``tool_result`` blocks); the live ``claude-code-api``
|
|
session pool stays the source of truth for *fingerprints*, and the DB
|
|
mirrors what we'd want to re-seed if a session evicts. See
|
|
``core/conversation_store.py`` for the diff-and-fork logic and
|
|
``frontends/markdown/frontend.py`` for the integration point.
|
|
|
|
Datetimes are stored UTC; we set ``default_factory`` rather than relying
|
|
on DB defaults so SQLite + Postgres behave identically. Every row that
|
|
needs an id uses ``Optional[int]`` so SQLAlchemy can autoincrement.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from sqlalchemy import JSON, Column, Index, UniqueConstraint, text
|
|
from sqlalchemy.dialects.postgresql import JSONB
|
|
from sqlmodel import Field, SQLModel
|
|
|
|
|
|
def _utcnow() -> datetime:
|
|
return datetime.now(UTC)
|
|
|
|
|
|
class Token(SQLModel, table=True):
|
|
"""Bearer token issued to an external caller.
|
|
|
|
``hashed_value`` holds the Argon2 hash (Phase 4.2 — until then,
|
|
rows are written by tests / the admin UI, not by ``TokenStore``).
|
|
Plaintext is shown to the user **once** at creation and then
|
|
discarded.
|
|
"""
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
name: str = Field(index=True, unique=True)
|
|
scope: str = Field(default="*")
|
|
hashed_value: str
|
|
created_at: datetime = Field(default_factory=_utcnow)
|
|
last_used_at: datetime | None = Field(default=None)
|
|
revoked_at: datetime | None = Field(default=None)
|
|
|
|
|
|
class AuditLog(SQLModel, table=True):
|
|
"""Append-only record of who-did-what.
|
|
|
|
``actor`` is ``"token:<name>"`` for inbound traffic or
|
|
``"admin:<user>"`` for admin-UI actions. ``kind`` is a short tag
|
|
(``"messages"`` / ``"mcp_call"`` / ``"token_create"`` / …);
|
|
free-form rather than enum so we can add new kinds without a
|
|
schema migration. ``detail_json`` is a JSON-encoded blob — keep
|
|
it small (paths, method, status), not full bodies.
|
|
"""
|
|
|
|
__tablename__ = "audit_log"
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
ts: datetime = Field(default_factory=_utcnow, index=True)
|
|
actor: str = Field(index=True)
|
|
kind: str = Field(index=True)
|
|
agent_name: str | None = Field(default=None, index=True)
|
|
detail_json: str = Field(default="{}")
|
|
|
|
|
|
class Conversation(SQLModel, table=True):
|
|
"""One conversation (§3.1): a master thread, a branch, a deep chat or a job.
|
|
|
|
``external_id`` is the public id (uuid) every frontend, the API and the
|
|
usage table refer to; ``frontend`` names the frontend that created the
|
|
row (``markdown``, ``api``, ``mcp``, ``system``). Where a conversation
|
|
is *visible* lives in :class:`ConversationBinding`.
|
|
|
|
``running_turn`` survives a restart: a non-null value at startup means
|
|
the gateway died mid-turn and the transcript needs its open ``tool_use``
|
|
closed before the session is resumed (``core/conversations``).
|
|
"""
|
|
|
|
__tablename__ = "conversations"
|
|
__table_args__ = (
|
|
UniqueConstraint("frontend", "external_id", name="uq_conv_frontend_extid"),
|
|
)
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
frontend: str = Field(index=True)
|
|
external_id: str = Field(index=True)
|
|
agent_name: str = Field(index=True)
|
|
session_id: str | None = Field(default=None, index=True)
|
|
kind: str = Field(default="deep", index=True)
|
|
parent_id: int | None = Field(default=None, index=True)
|
|
title: str | None = Field(default=None)
|
|
status: str = Field(default="open", index=True)
|
|
running_turn: str | None = Field(default=None)
|
|
pending_question: bool = Field(default=False)
|
|
flags: dict[str, Any] = Field(
|
|
default_factory=dict,
|
|
sa_column=Column(
|
|
JSON().with_variant(JSONB(), "postgresql"),
|
|
nullable=False,
|
|
server_default=text("'{}'"),
|
|
),
|
|
)
|
|
created_at: datetime = Field(default_factory=_utcnow)
|
|
updated_at: datetime = Field(default_factory=_utcnow)
|
|
last_user_activity_at: datetime | None = Field(default=None)
|
|
last_activity_at: datetime | None = Field(default=None)
|
|
|
|
|
|
class ConversationBinding(SQLModel, table=True):
|
|
"""Where a conversation shows up: a Telegram topic id, a vault-relative path.
|
|
|
|
Invariant (§3.1): at most one *visible* binding per frontend per
|
|
conversation - enforced by the partial unique index. The same external
|
|
id may point at several conversations over time (a renamed topic gets a
|
|
new branch), only one of them visible.
|
|
"""
|
|
|
|
__tablename__ = "conversation_bindings"
|
|
__table_args__ = (
|
|
Index(
|
|
"uq_binding_visible",
|
|
"conversation_id",
|
|
"frontend",
|
|
unique=True,
|
|
postgresql_where=text("visible"),
|
|
sqlite_where=text("visible"),
|
|
),
|
|
Index("ix_binding_lookup", "frontend", "external_id"),
|
|
)
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
conversation_id: int = Field(index=True)
|
|
frontend: str
|
|
external_id: str
|
|
visible: bool = Field(default=True)
|
|
created_at: datetime = Field(default_factory=_utcnow)
|
|
|
|
|
|
class InjectQueueItem(SQLModel, table=True):
|
|
"""Persisted per-conversation queue (§3.4), ``urgent > user > wake > normal``.
|
|
|
|
``status`` walks ``queued -> running -> done``; a row still ``running``
|
|
at startup was cut by a restart and becomes ``interrupted`` - it is
|
|
never re-run, the agent gets an inject saying so instead.
|
|
"""
|
|
|
|
__tablename__ = "inject_queue"
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
conversation_id: int = Field(index=True)
|
|
priority: str = Field(index=True)
|
|
origin: str = Field(default="system")
|
|
text: str
|
|
status: str = Field(default="queued", index=True)
|
|
turn_id: str | None = Field(default=None)
|
|
interrupted_turn: bool | None = Field(default=None)
|
|
"""An urgent inject that cut a running turn; the header tells the model
|
|
the ``Request interrupted`` tool result above was this, not a refusal."""
|
|
created_at: datetime = Field(default_factory=_utcnow)
|
|
delivered_at: datetime | None = Field(default=None)
|
|
|
|
|
|
class TelegramUpdate(SQLModel, table=True):
|
|
"""Inbox of the Telegram frontend (§3.8).
|
|
|
|
The update is stored before the poll offset moves past it and handled
|
|
from here, so a restart neither loses nor duplicates a message.
|
|
"""
|
|
|
|
__tablename__ = "telegram_updates"
|
|
|
|
update_id: int = Field(primary_key=True, sa_column_kwargs={"autoincrement": False})
|
|
payload: dict[str, Any] = Field(
|
|
sa_column=Column(JSON().with_variant(JSONB(), "postgresql"), nullable=False)
|
|
)
|
|
received_at: datetime = Field(default_factory=_utcnow)
|
|
processed_at: datetime | None = Field(default=None, index=True)
|
|
error: str | None = Field(default=None)
|
|
|
|
|
|
class Delivery(SQLModel, table=True):
|
|
"""Outbox (§3.8): a reply is a row first and a Telegram message second.
|
|
|
|
``status`` walks ``queued -> sent`` with retries on the way, ``failed``
|
|
only when Telegram rejects the row for good (unknown thread, blocked
|
|
bot). ``dedupe_key`` keeps one row per turn chunk across restarts.
|
|
"""
|
|
|
|
__tablename__ = "deliveries"
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
conversation_id: int | None = Field(default=None, index=True)
|
|
frontend: str = Field(default="telegram", index=True)
|
|
chat_id: int
|
|
thread_id: int | None = Field(default=None)
|
|
text: str
|
|
plain: bool = Field(default=False)
|
|
turn_id: str | None = Field(default=None)
|
|
dedupe_key: str | None = Field(default=None, unique=True)
|
|
status: str = Field(default="queued", index=True)
|
|
attempts: int = Field(default=0)
|
|
next_attempt_at: datetime = Field(default_factory=_utcnow, index=True)
|
|
last_error: str | None = Field(default=None)
|
|
message_id: int | None = Field(default=None)
|
|
created_at: datetime = Field(default_factory=_utcnow)
|
|
sent_at: datetime | None = Field(default=None)
|
|
|
|
|
|
class ConversationMessage(SQLModel, table=True):
|
|
"""One raw Anthropic-shape message in a conversation's transcript.
|
|
|
|
A single user/assistant exchange visible in Obsidian can occupy
|
|
multiple rows when claude ran a tool cycle: ``assistant``
|
|
(tool_use), ``user`` (tool_result), ``assistant`` (final text) all
|
|
live as separate rows with the same conversation_id and monotonic
|
|
``seq``. ``content_json`` is the canonical Anthropic content payload
|
|
(string or list-of-blocks) — exactly what we'll feed back to the
|
|
backend so its session-pool fingerprint matches.
|
|
|
|
``seq`` is per-conversation 0-based monotonic; the unique
|
|
constraint catches the trivial bug of two writers racing on the
|
|
same conversation.
|
|
"""
|
|
|
|
__tablename__ = "conversation_messages"
|
|
__table_args__ = (
|
|
UniqueConstraint("conversation_id", "seq", name="uq_msg_conv_seq"),
|
|
)
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
conversation_id: int = Field(index=True)
|
|
seq: int
|
|
role: str
|
|
content_json: str
|
|
created_at: datetime = Field(default_factory=_utcnow)
|
|
|
|
|
|
class TranscriptEntry(SQLModel, table=True):
|
|
"""One Agent SDK transcript line, mirrored from the session store protocol.
|
|
|
|
``(project_key, session_id, subpath)`` is the ``SessionKey``; ``subpath``
|
|
is ``""`` for the main transcript so the unique constraints stay simple.
|
|
``uuid`` is the SDK's idempotency key - the partial unique index rejects
|
|
a replayed batch, entries without a uuid are appended as-is.
|
|
"""
|
|
|
|
__tablename__ = "transcript_entries"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"project_key", "session_id", "subpath", "seq", name="uq_transcript_seq"
|
|
),
|
|
Index(
|
|
"uq_transcript_uuid",
|
|
"project_key",
|
|
"session_id",
|
|
"subpath",
|
|
"uuid",
|
|
unique=True,
|
|
postgresql_where=text("uuid IS NOT NULL"),
|
|
sqlite_where=text("uuid IS NOT NULL"),
|
|
),
|
|
)
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
project_key: str = Field(index=True)
|
|
session_id: str = Field(index=True)
|
|
subpath: str = Field(default="")
|
|
seq: int
|
|
uuid: str | None = Field(default=None)
|
|
entry: dict[str, Any] = Field(
|
|
sa_column=Column(JSON().with_variant(JSONB(), "postgresql"), nullable=False)
|
|
)
|
|
origin: str | None = Field(default=None)
|
|
source: str | None = Field(default=None)
|
|
turn_id: str | None = Field(default=None)
|
|
created_at: datetime = Field(default_factory=_utcnow)
|
|
|
|
|
|
class Usage(SQLModel, table=True):
|
|
"""Per-turn token accounting from ``ResultMessage``.
|
|
|
|
Token columns come from ``ResultMessage.usage`` and are sums over the
|
|
turn's API calls. ``cost_usd`` and ``model_usage`` are **per-turn deltas**:
|
|
the SDK reports ``total_cost_usd`` and ``model_usage`` cumulatively for
|
|
the claude process (growing across the turns of a session, reset when
|
|
the process is respawned), so :func:`append_usage` stores the raw values
|
|
in ``cost_usd_total`` / ``model_usage_total`` and the difference against
|
|
the previous row of the same session here. Summing ``cost_usd`` over rows
|
|
is therefore meaningful; summing ``cost_usd_total`` is not.
|
|
"""
|
|
|
|
__tablename__ = "usage"
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
ts: datetime = Field(default_factory=_utcnow, index=True)
|
|
agent_name: str = Field(index=True)
|
|
conversation_id: str | None = Field(default=None, index=True)
|
|
session_id: str | None = Field(default=None, index=True)
|
|
model: str
|
|
effort: str | None = Field(default=None)
|
|
input_tokens: int = 0
|
|
output_tokens: int = 0
|
|
cache_read_tokens: int = 0
|
|
cache_creation_tokens: int = 0
|
|
context_tokens: int | None = Field(default=None)
|
|
"""Input of the turn's last API call - the real context size."""
|
|
cost_usd: float | None = Field(default=None)
|
|
duration_ms: int | None = Field(default=None)
|
|
num_turns: int | None = Field(default=None)
|
|
model_usage: dict[str, Any] | None = Field(
|
|
default=None,
|
|
sa_column=Column(JSON().with_variant(JSONB(), "postgresql"), nullable=True),
|
|
)
|
|
"""Per-model tokens, cost and web searches - per-turn delta (see class doc)."""
|
|
cost_usd_total: float | None = Field(default=None)
|
|
"""``ResultMessage.total_cost_usd`` verbatim: cumulative for the claude process."""
|
|
model_usage_total: dict[str, Any] | None = Field(
|
|
default=None,
|
|
sa_column=Column(JSON().with_variant(JSONB(), "postgresql"), nullable=True),
|
|
)
|
|
"""``ResultMessage.model_usage`` verbatim: cumulative for the claude process."""
|
|
|
|
|
|
class RateLimit(SQLModel, table=True):
|
|
"""One ``RateLimitEvent`` from the SDK; the newest row per ``window`` is the state.
|
|
|
|
The quota is per subscription, not per agent or conversation - the row
|
|
only remembers which session reported it.
|
|
"""
|
|
|
|
__tablename__ = "rate_limits"
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
ts: datetime = Field(default_factory=_utcnow, index=True)
|
|
window: str = Field(index=True)
|
|
status: str
|
|
utilization: float | None = Field(default=None)
|
|
resets_at: datetime | None = Field(default=None)
|
|
overage_status: str | None = Field(default=None)
|
|
overage_resets_at: datetime | None = Field(default=None)
|
|
agent_name: str | None = Field(default=None)
|
|
session_id: str | None = Field(default=None)
|
|
raw: dict[str, Any] = Field(
|
|
default_factory=dict,
|
|
sa_column=Column(
|
|
JSON().with_variant(JSONB(), "postgresql"),
|
|
nullable=False,
|
|
server_default=text("'{}'"),
|
|
),
|
|
)
|
|
|
|
|
|
class JobRunRecord(SQLModel, table=True):
|
|
"""One finished run of a scheduler job (§3.6): what fired it, how it ended.
|
|
|
|
``error`` keeps the head of the traceback of a failed run; ``payload``
|
|
is the trigger's data (``{}`` for cron) and should stay small.
|
|
"""
|
|
|
|
__tablename__ = "job_runs"
|
|
|
|
id: int | None = Field(default=None, primary_key=True)
|
|
job: str = Field(index=True)
|
|
trigger: str
|
|
started_at: datetime
|
|
finished_at: datetime
|
|
status: str
|
|
error: str | None = Field(default=None)
|
|
payload: dict[str, Any] = Field(
|
|
default_factory=dict,
|
|
sa_column=Column(
|
|
JSON().with_variant(JSONB(), "postgresql"),
|
|
nullable=False,
|
|
server_default=text("'{}'"),
|
|
),
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"AuditLog",
|
|
"Conversation",
|
|
"ConversationBinding",
|
|
"ConversationMessage",
|
|
"Delivery",
|
|
"InjectQueueItem",
|
|
"JobRunRecord",
|
|
"RateLimit",
|
|
"TelegramUpdate",
|
|
"Token",
|
|
"TranscriptEntry",
|
|
"Usage",
|
|
]
|