feat(bot,frontend,backend): add integration with external collaborative solver

frontend/src/lib/components/ChatMessage.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
-	import { Marked } from 'marked';
 	import LoadingDots from './LoadingDots.svelte';
+	import { processContent } from '$lib/utils/markdown';
 
 	interface Props {
 		role: 'user' | 'assistant';
@@ -9,28 +9,6 @@
 	}
 
 	let { role, content, isStreaming = false }: Props = $props();
-
-	const marked = new Marked({
-		breaks: true,
-		gfm: true
-	});
-
-	function processLatex(text: string): string {
-		return text
-			.replace(/\$\$(.*?)\$\$/gs, (_, tex) => {
-				const encoded = encodeURIComponent(tex.trim());
-				return `<img src="/service/latex?tex=${encoded}&display=1" alt="LaTeX" class="block my-1 max-h-12" />`;
-			})
-			.replace(/\$(.+?)\$/g, (_, tex) => {
-				const encoded = encodeURIComponent(tex.trim());
-				return `<img src="/service/latex?tex=${encoded}" alt="LaTeX" class="inline-block align-middle max-h-4" />`;
-			});
-	}
-
-	function processContent(text: string): string {
-		const withLatex = processLatex(text);
-		return marked.parse(withLatex) as string;
-	}
 </script>
 
 <div

frontend/src/lib/components/IncomingSheetsPanel.svelte

@@ -0,0 +1,53 @@
+<script lang="ts">
+	import type { Id } from '$lib/convex/_generated/dataModel';
+	import { processContent } from '$lib/utils/markdown';
+
+	interface Sheet {
+		_id: Id<'incomingSheets'>;
+		sheetId: number;
+		text: string;
+	}
+
+	interface Props {
+		sheets: Sheet[];
+		onaccept: (id: Id<'incomingSheets'>) => void;
+		ondismiss: (id: Id<'incomingSheets'>) => void;
+	}
+
+	let { sheets, onaccept, ondismiss }: Props = $props();
+</script>
+
+{#if sheets.length > 0}
+	<div class="space-y-1.5">
+		<div class="text-[8px] tracking-wider text-neutral-500 uppercase">
+			incoming sheets · {sheets.length}
+		</div>
+		{#each sheets as s (s._id)}
+			<div class="rounded border border-neutral-800 bg-neutral-900/60 p-2">
+				<div class="text-[8px] text-neutral-500">sheet #{s.sheetId}</div>
+				<div
+					class="prose-mini mt-1 max-h-64 overflow-y-auto text-[10px] leading-relaxed text-neutral-200"
+				>
+					<!-- eslint-disable-next-line svelte/no-at-html-tags -->
+					{@html processContent(s.text)}
+				</div>
+				<div class="mt-1.5 flex gap-1">
+					<button
+						type="button"
+						onclick={() => onaccept(s._id)}
+						class="flex-1 rounded bg-blue-600 py-1 text-[9px] text-white"
+					>
+						accept
+					</button>
+					<button
+						type="button"
+						onclick={() => ondismiss(s._id)}
+						class="flex-1 rounded bg-neutral-800 py-1 text-[9px] text-neutral-400"
+					>
+						dismiss
+					</button>
+				</div>
+			</div>
+		{/each}
+	</div>
+{/if}

frontend/src/lib/convex/_generated/api.d.ts

@@ -9,8 +9,10 @@
  */
 
 import type * as chats from "../chats.js";
+import type * as collaborative from "../collaborative.js";
 import type * as devicePairings from "../devicePairings.js";
 import type * as http from "../http.js";
+import type * as incomingSheets from "../incomingSheets.js";
 import type * as inject from "../inject.js";
 import type * as injectConnections from "../injectConnections.js";
 import type * as messages from "../messages.js";
@@ -30,8 +32,10 @@ import type {
 
 declare const fullApi: ApiFromModules<{
   chats: typeof chats;
+  collaborative: typeof collaborative;
   devicePairings: typeof devicePairings;
   http: typeof http;
+  incomingSheets: typeof incomingSheets;
   inject: typeof inject;
   injectConnections: typeof injectConnections;
   messages: typeof messages;

frontend/src/lib/convex/chats.ts

@@ -86,6 +86,21 @@ export const getWithUser = query({
 						ragDatabaseId: v.id('ragDatabases'),
 						activeSince: v.number()
 					})
+				),
+				injectCollectionMode: v.optional(
+					v.object({
+						injectDatabaseId: v.id('injectDatabases'),
+						activeSince: v.number()
+					})
+				),
+				collaborativeRoom: v.optional(
+					v.object({
+						pin: v.string(),
+						creatorToken: v.optional(v.string()),
+						joinedAt: v.number(),
+						lastSeenHistoryCount: v.optional(v.number()),
+						lastSeenSheetId: v.optional(v.number())
+					})
 				)
 			})
 		}),

frontend/src/lib/convex/collaborative.ts

@@ -0,0 +1,148 @@
+import { v } from 'convex/values';
+import { mutation, query } from './_generated/server';
+
+const PENDING_UPLOAD_RETURN = v.object({
+	_id: v.id('pendingRoomUploads'),
+	_creationTime: v.number(),
+	userId: v.id('users'),
+	pin: v.string(),
+	imageBase64: v.string(),
+	mediaType: v.string(),
+	status: v.union(
+		v.literal('pending'),
+		v.literal('uploading'),
+		v.literal('done'),
+		v.literal('failed')
+	),
+	attempts: v.number(),
+	lastError: v.optional(v.string()),
+	createdAt: v.number(),
+	updatedAt: v.number()
+});
+
+export const enqueueUpload = mutation({
+	args: {
+		userId: v.id('users'),
+		pin: v.string(),
+		imageBase64: v.string(),
+		mediaType: v.string()
+	},
+	returns: v.id('pendingRoomUploads'),
+	handler: async (ctx, args) => {
+		const now = Date.now();
+		return await ctx.db.insert('pendingRoomUploads', {
+			userId: args.userId,
+			pin: args.pin,
+			imageBase64: args.imageBase64,
+			mediaType: args.mediaType,
+			status: 'pending',
+			attempts: 0,
+			createdAt: now,
+			updatedAt: now
+		});
+	}
+});
+
+export const listPendingUploads = query({
+	args: {},
+	returns: v.array(PENDING_UPLOAD_RETURN),
+	handler: async (ctx) => {
+		const now = Date.now();
+		const items = await ctx.db
+			.query('pendingRoomUploads')
+			.withIndex('by_status_and_updated_at', (q) => q.eq('status', 'pending'))
+			.collect();
+		return items.filter((i) => i.updatedAt <= now);
+	}
+});
+
+export const markUploading = mutation({
+	args: { id: v.id('pendingRoomUploads') },
+	returns: v.boolean(),
+	handler: async (ctx, args) => {
+		const item = await ctx.db.get(args.id);
+		if (!item || item.status !== 'pending') return false;
+		await ctx.db.patch(args.id, { status: 'uploading', updatedAt: Date.now() });
+		return true;
+	}
+});
+
+export const markDone = mutation({
+	args: { id: v.id('pendingRoomUploads') },
+	returns: v.null(),
+	handler: async (ctx, args) => {
+		await ctx.db.delete(args.id);
+		return null;
+	}
+});
+
+export const bumpAttempt = mutation({
+	args: {
+		id: v.id('pendingRoomUploads'),
+		error: v.string(),
+		nextRetryAt: v.number()
+	},
+	returns: v.null(),
+	handler: async (ctx, args) => {
+		const item = await ctx.db.get(args.id);
+		if (!item) return null;
+		await ctx.db.patch(args.id, {
+			status: 'pending',
+			attempts: item.attempts + 1,
+			lastError: args.error,
+			updatedAt: args.nextRetryAt
+		});
+		return null;
+	}
+});
+
+export const markFailed = mutation({
+	args: { id: v.id('pendingRoomUploads'), error: v.string() },
+	returns: v.null(),
+	handler: async (ctx, args) => {
+		const item = await ctx.db.get(args.id);
+		if (!item) return null;
+		await ctx.db.patch(args.id, {
+			status: 'failed',
+			lastError: args.error,
+			updatedAt: Date.now()
+		});
+		return null;
+	}
+});
+
+export const resetStuckUploads = mutation({
+	args: {},
+	returns: v.number(),
+	handler: async (ctx) => {
+		const stuck = await ctx.db
+			.query('pendingRoomUploads')
+			.withIndex('by_status_and_updated_at', (q) => q.eq('status', 'uploading'))
+			.collect();
+		const now = Date.now();
+		for (const item of stuck) {
+			await ctx.db.patch(item._id, { status: 'pending', updatedAt: now });
+		}
+		return stuck.length;
+	}
+});
+
+export const queueDepthForUser = query({
+	args: { userId: v.id('users') },
+	returns: v.object({
+		pending: v.number(),
+		uploading: v.number(),
+		failed: v.number()
+	}),
+	handler: async (ctx, args) => {
+		const all = await ctx.db
+			.query('pendingRoomUploads')
+			.withIndex('by_user_id', (q) => q.eq('userId', args.userId))
+			.collect();
+		return {
+			pending: all.filter((i) => i.status === 'pending').length,
+			uploading: all.filter((i) => i.status === 'uploading').length,
+			failed: all.filter((i) => i.status === 'failed').length
+		};
+	}
+});

frontend/src/lib/convex/incomingSheets.ts

@@ -0,0 +1,121 @@
+import { v } from 'convex/values';
+import { mutation, query } from './_generated/server';
+
+const SHEET_RETURN = v.object({
+	_id: v.id('incomingSheets'),
+	_creationTime: v.number(),
+	userId: v.id('users'),
+	chatId: v.id('chats'),
+	pin: v.string(),
+	sheetId: v.number(),
+	sheetCreatedAt: v.number(),
+	text: v.string(),
+	status: v.union(v.literal('preview'), v.literal('accepted'), v.literal('dismissed')),
+	linkedMessageId: v.optional(v.id('messages')),
+	acceptedAt: v.optional(v.number()),
+	createdAt: v.number()
+});
+
+export const createMany = mutation({
+	args: {
+		userId: v.id('users'),
+		chatId: v.id('chats'),
+		pin: v.string(),
+		sheets: v.array(
+			v.object({
+				sheetId: v.number(),
+				sheetCreatedAt: v.number(),
+				text: v.string()
+			})
+		)
+	},
+	returns: v.number(),
+	handler: async (ctx, args) => {
+		const now = Date.now();
+		let inserted = 0;
+		for (const s of args.sheets) {
+			const existing = await ctx.db
+				.query('incomingSheets')
+				.withIndex('by_chat_id_and_sheet_id', (q) =>
+					q.eq('chatId', args.chatId).eq('sheetId', s.sheetId)
+				)
+				.unique();
+			if (existing) continue;
+			await ctx.db.insert('incomingSheets', {
+				userId: args.userId,
+				chatId: args.chatId,
+				pin: args.pin,
+				sheetId: s.sheetId,
+				sheetCreatedAt: s.sheetCreatedAt,
+				text: s.text,
+				status: 'preview',
+				createdAt: now
+			});
+			inserted += 1;
+		}
+		return inserted;
+	}
+});
+
+export const listForChat = query({
+	args: {
+		chatId: v.id('chats'),
+		status: v.union(v.literal('preview'), v.literal('accepted'), v.literal('dismissed'))
+	},
+	returns: v.array(SHEET_RETURN),
+	handler: async (ctx, args) => {
+		return await ctx.db
+			.query('incomingSheets')
+			.withIndex('by_chat_id_and_status', (q) =>
+				q.eq('chatId', args.chatId).eq('status', args.status)
+			)
+			.order('asc')
+			.collect();
+	}
+});
+
+export const accept = mutation({
+	args: { id: v.id('incomingSheets') },
+	returns: v.union(v.id('messages'), v.null()),
+	handler: async (ctx, args) => {
+		const sheet = await ctx.db.get(args.id);
+		if (!sheet || sheet.status !== 'preview') return null;
+
+		const messageId = await ctx.db.insert('messages', {
+			chatId: sheet.chatId,
+			role: 'user',
+			content: sheet.text,
+			source: 'web',
+			createdAt: Date.now()
+		});
+
+		const chat = await ctx.db.get(sheet.chatId);
+		if (chat) {
+			await ctx.db.insert('pendingGenerations', {
+				userId: chat.userId,
+				chatId: sheet.chatId,
+				userMessage: sheet.text,
+				createdAt: Date.now()
+			});
+		}
+
+		await ctx.db.patch(args.id, {
+			status: 'accepted',
+			linkedMessageId: messageId,
+			acceptedAt: Date.now()
+		});
+
+		return messageId;
+	}
+});
+
+export const dismiss = mutation({
+	args: { id: v.id('incomingSheets') },
+	returns: v.null(),
+	handler: async (ctx, args) => {
+		const sheet = await ctx.db.get(args.id);
+		if (!sheet || sheet.status !== 'preview') return null;
+		await ctx.db.patch(args.id, { status: 'dismissed' });
+		return null;
+	}
+});

frontend/src/lib/convex/messages.ts

@@ -146,6 +146,24 @@ export const update = mutation({
 	}
 });
 
+export const setTelegramMessageId = mutation({
+	args: { messageId: v.id('messages'), telegramMessageId: v.number() },
+	returns: v.null(),
+	handler: async (ctx, args) => {
+		await ctx.db.patch(args.messageId, { telegramMessageId: args.telegramMessageId });
+		return null;
+	}
+});
+
+export const getTelegramMessageId = query({
+	args: { messageId: v.id('messages') },
+	returns: v.union(v.number(), v.null()),
+	handler: async (ctx, args) => {
+		const msg = await ctx.db.get(args.messageId);
+		return msg?.telegramMessageId ?? null;
+	}
+});
+
 export const getHistoryForAI = query({
 	args: { chatId: v.id('chats'), limit: v.optional(v.number()) },
 	returns: v.array(

frontend/src/lib/convex/schema.ts

@@ -22,6 +22,15 @@ export default defineSchema({
 				injectDatabaseId: v.id('injectDatabases'),
 				activeSince: v.number()
 			})
+		),
+		collaborativeRoom: v.optional(
+			v.object({
+				pin: v.string(),
+				creatorToken: v.optional(v.string()),
+				joinedAt: v.number(),
+				lastSeenHistoryCount: v.optional(v.number()),
+				lastSeenSheetId: v.optional(v.number())
+			})
 		)
 	}).index('by_telegram_id', ['telegramId']),
 
@@ -43,7 +52,8 @@ export default defineSchema({
 		followUpOptions: v.optional(v.array(v.string())),
 		source: v.union(v.literal('telegram'), v.literal('web')),
 		createdAt: v.number(),
-		isStreaming: v.optional(v.boolean())
+		isStreaming: v.optional(v.boolean()),
+		telegramMessageId: v.optional(v.number())
 	})
 		.index('by_chat_id', ['chatId'])
 		.index('by_chat_id_and_created_at', ['chatId', 'createdAt']),
@@ -147,5 +157,58 @@ export default defineSchema({
 	})
 		.index('by_user_id', ['userId'])
 		.index('by_user_id_and_inject_database_id', ['userId', 'injectDatabaseId'])
-		.index('by_inject_database_id', ['injectDatabaseId'])
+		.index('by_inject_database_id', ['injectDatabaseId']),
+
+	pendingRoomUploads: defineTable({
+		userId: v.id('users'),
+		pin: v.string(),
+		imageBase64: v.string(),
+		mediaType: v.string(),
+		status: v.union(
+			v.literal('pending'),
+			v.literal('uploading'),
+			v.literal('done'),
+			v.literal('failed')
+		),
+		attempts: v.number(),
+		lastError: v.optional(v.string()),
+		createdAt: v.number(),
+		updatedAt: v.number()
+	})
+		.index('by_status_and_updated_at', ['status', 'updatedAt'])
+		.index('by_user_id', ['userId']),
+
+	incomingSolutions: defineTable({
+		userId: v.id('users'),
+		chatId: v.id('chats'),
+		pin: v.string(),
+		conditionSnippet: v.string(),
+		problem: v.string(),
+		snippet: v.string(),
+		fullSolution: v.string(),
+		plain: v.string(),
+		status: v.union(v.literal('preview'), v.literal('accepted'), v.literal('dismissed')),
+		linkedMessageId: v.optional(v.id('messages')),
+		telegramMessageId: v.optional(v.number()),
+		acceptedAt: v.optional(v.number()),
+		createdAt: v.number()
+	})
+		.index('by_chat_id_and_status', ['chatId', 'status'])
+		.index('by_status_and_telegram_message_id', ['status', 'telegramMessageId'])
+		.index('by_linked_message_id', ['linkedMessageId']),
+
+	incomingSheets: defineTable({
+		userId: v.id('users'),
+		chatId: v.id('chats'),
+		pin: v.string(),
+		sheetId: v.number(),
+		sheetCreatedAt: v.number(),
+		text: v.string(),
+		status: v.union(v.literal('preview'), v.literal('accepted'), v.literal('dismissed')),
+		linkedMessageId: v.optional(v.id('messages')),
+		acceptedAt: v.optional(v.number()),
+		createdAt: v.number()
+	})
+		.index('by_chat_id_and_status', ['chatId', 'status'])
+		.index('by_chat_id_and_sheet_id', ['chatId', 'sheetId'])
 });

frontend/src/lib/convex/users.ts

@@ -1,5 +1,6 @@
 import { v } from 'convex/values';
 import { mutation, query } from './_generated/server';
+import type { Id } from './_generated/dataModel';
 
 const DEFAULT_MODEL = 'gemini-3-pro-preview';
 
@@ -28,6 +29,15 @@ export const getById = query({
 					injectDatabaseId: v.id('injectDatabases'),
 					activeSince: v.number()
 				})
+			),
+			collaborativeRoom: v.optional(
+				v.object({
+					pin: v.string(),
+					creatorToken: v.optional(v.string()),
+					joinedAt: v.number(),
+					lastSeenHistoryCount: v.optional(v.number()),
+					lastSeenSheetId: v.optional(v.number())
+				})
 			)
 		}),
 		v.null()
@@ -62,6 +72,15 @@ export const getByTelegramId = query({
 					injectDatabaseId: v.id('injectDatabases'),
 					activeSince: v.number()
 				})
+			),
+			collaborativeRoom: v.optional(
+				v.object({
+					pin: v.string(),
+					creatorToken: v.optional(v.string()),
+					joinedAt: v.number(),
+					lastSeenHistoryCount: v.optional(v.number()),
+					lastSeenSheetId: v.optional(v.number())
+				})
 			)
 		}),
 		v.null()
@@ -227,3 +246,71 @@ export const getInjectCollectionMode = query({
 		return user?.injectCollectionMode ?? null;
 	}
 });
+
+export const setCollaborativeRoom = mutation({
+	args: {
+		userId: v.id('users'),
+		pin: v.string(),
+		creatorToken: v.optional(v.string()),
+		lastSeenSheetId: v.optional(v.number())
+	},
+	returns: v.null(),
+	handler: async (ctx, args) => {
+		await ctx.db.patch(args.userId, {
+			collaborativeRoom: {
+				pin: args.pin,
+				creatorToken: args.creatorToken,
+				joinedAt: Date.now(),
+				lastSeenSheetId: args.lastSeenSheetId ?? 0
+			}
+		});
+		return null;
+	}
+});
+
+export const clearCollaborativeRoom = mutation({
+	args: { userId: v.id('users') },
+	returns: v.null(),
+	handler: async (ctx, args) => {
+		await ctx.db.patch(args.userId, { collaborativeRoom: undefined });
+		return null;
+	}
+});
+
+export const setLastSeenSheetId = mutation({
+	args: { userId: v.id('users'), sheetId: v.number() },
+	returns: v.null(),
+	handler: async (ctx, args) => {
+		const user = await ctx.db.get(args.userId);
+		if (!user?.collaborativeRoom) return null;
+		await ctx.db.patch(args.userId, {
+			collaborativeRoom: { ...user.collaborativeRoom, lastSeenSheetId: args.sheetId }
+		});
+		return null;
+	}
+});
+
+export const listActiveRoomUsers = query({
+	args: {},
+	returns: v.array(
+		v.object({
+			_id: v.id('users'),
+			pin: v.string(),
+			lastSeenSheetId: v.number()
+		})
+	),
+	handler: async (ctx) => {
+		const users = await ctx.db.query('users').collect();
+		const result: Array<{ _id: Id<'users'>; pin: string; lastSeenSheetId: number }> = [];
+		for (const u of users) {
+			if (u.collaborativeRoom) {
+				result.push({
+					_id: u._id,
+					pin: u.collaborativeRoom.pin,
+					lastSeenSheetId: u.collaborativeRoom.lastSeenSheetId ?? 0
+				});
+			}
+		}
+		return result;
+	}
+});

frontend/src/lib/utils/markdown.ts

@@ -0,0 +1,19 @@
+import { Marked } from 'marked';
+
+const marked = new Marked({ breaks: true, gfm: true });
+
+export function processLatex(text: string): string {
+	return text
+		.replace(/\$\$(.*?)\$\$/gs, (_, tex) => {
+			const encoded = encodeURIComponent(tex.trim());
+			return `<img src="/service/latex?tex=${encoded}&display=1" alt="LaTeX" class="block my-1 max-h-12" />`;
+		})
+		.replace(/\$(.+?)\$/g, (_, tex) => {
+			const encoded = encodeURIComponent(tex.trim());
+			return `<img src="/service/latex?tex=${encoded}" alt="LaTeX" class="inline-block align-middle max-h-4" />`;
+		});
+}
+
+export function processContent(text: string): string {
+	return marked.parse(processLatex(text)) as string;
+}

frontend/src/routes/[mnemonic]/+page.svelte

@@ -20,6 +20,7 @@
 	import PhotoPreview from '$lib/components/PhotoPreview.svelte';
 	import DraftBadge from '$lib/components/DraftBadge.svelte';
 	import SilentCapture from '$lib/components/SilentCapture.svelte';
+	import IncomingSheetsPanel from '$lib/components/IncomingSheetsPanel.svelte';
 
 	const usePolling = getContext<boolean>('convex-use-polling') ?? false;
 	let mnemonic = $derived(page.params.mnemonic);
@@ -193,6 +194,38 @@
 	const photoDraft = $derived(usePolling ? photoDraftPoll! : photoDraftWs!);
 	const draftPhotos = $derived(photoDraft.data?.photos ?? []);
 
+	const incomingSheetsWs = usePolling
+		? null
+		: useQuery(api.incomingSheets.listForChat, () =>
+				chatId ? { chatId, status: 'preview' as const } : 'skip'
+			);
+	const incomingSheetsPoll = usePolling
+		? usePollingQuery(api.incomingSheets.listForChat, () =>
+				chatId ? { chatId, status: 'preview' as const } : 'skip'
+			)
+		: null;
+	const incomingSheetsQ = $derived(usePolling ? incomingSheetsPoll! : incomingSheetsWs!);
+	const incomingSheetItems = $derived(incomingSheetsQ.data ?? []);
+
+	const acceptSheetPoll = usePolling ? usePollingMutation(api.incomingSheets.accept) : null;
+	const dismissSheetPoll = usePolling ? usePollingMutation(api.incomingSheets.dismiss) : null;
+
+	function handleAcceptSheet(id: Id<'incomingSheets'>) {
+		if (usePolling && acceptSheetPoll) {
+			acceptSheetPoll({ id });
+		} else if (clientWs) {
+			clientWs.mutation(api.incomingSheets.accept, { id });
+		}
+	}
+
+	function handleDismissSheet(id: Id<'incomingSheets'>) {
+		if (usePolling && dismissSheetPoll) {
+			dismissSheetPoll({ id });
+		} else if (clientWs) {
+			clientWs.mutation(api.incomingSheets.dismiss, { id });
+		}
+	}
+
 	$effect(() => {
 		const req = captureNowRequest.data;
 		if (req && hasCamera && !processedCaptureNowIds.has(req._id)) {
@@ -611,6 +644,13 @@
 			{#if draftPhotos.length > 0}
 				<DraftBadge photos={draftPhotos} onremove={handleRemoveDraftPhoto} />
 			{/if}
+			{#if incomingSheetItems.length > 0}
+				<IncomingSheetsPanel
+					sheets={incomingSheetItems}
+					onaccept={handleAcceptSheet}
+					ondismiss={handleDismissSheet}
+				/>
+			{/if}
 			<ChatInput onsubmit={sendMessage} allowEmpty={draftPhotos.length > 0} />
 		</div>
 	{/if}