feat(frontend): optimize photo requesting for slower internet

frontend/src/lib/components/DraftBadge.svelte

@@ -1,6 +1,8 @@
 <script lang="ts">
+	import type { Id } from '$lib/convex/_generated/dataModel';
+
 	interface Photo {
-		base64: string;
+		_id: Id<'photoDrafts'>;
 		mediaType: string;
 	}
 

frontend/src/lib/components/SilentCapture.svelte

@@ -3,9 +3,10 @@
 
 	interface Props {
 		oncapture: (base64: string, mediaType: string, thumbnailBase64: string) => void;
+		onunpair?: () => void;
 	}
 
-	let { oncapture }: Props = $props();
+	let { oncapture, onunpair }: Props = $props();
 
 	let videoElement: HTMLVideoElement | null = $state(null);
 	let stream: MediaStream | null = $state(null);
@@ -123,4 +124,12 @@
 <div class="fixed inset-0 z-40 bg-black">
 	<video bind:this={videoElement} autoplay playsinline muted class="h-full w-full object-cover"
 	></video>
+	{#if onunpair}
+		<button
+			onclick={onunpair}
+			class="absolute top-4 left-4 z-10 rounded-full bg-red-600/80 px-3 py-1.5 text-xs text-white"
+		>
+			unpair
+		</button>
+	{/if}
 </div>

frontend/src/lib/convex-polling.svelte.ts

@@ -3,7 +3,7 @@ import { getContext, setContext } from 'svelte';
 import type { FunctionReference, FunctionArgs, FunctionReturnType } from 'convex/server';
 
 const POLLING_CONTEXT_KEY = 'convex-polling';
-const POLL_INTERVAL = 500;
+const POLL_INTERVAL = 1000;
 
 type PollingContext = {
 	client: ConvexHttpClient;
@@ -47,7 +47,11 @@ export function usePollingMutation<Mutation extends FunctionReference<'mutation'
 export function usePollingQuery<Query extends FunctionReference<'query'>>(
 	query: Query,
 	argsGetter: () => FunctionArgs<Query> | 'skip'
-): { data: FunctionReturnType<Query> | undefined; error: Error | null; isLoading: boolean } {
+): {
+	data: FunctionReturnType<Query> | undefined;
+	error: Error | null;
+	isLoading: boolean;
+} {
 	const client = usePollingClient();
 
 	// eslint-disable-next-line prefer-const

frontend/src/lib/convex/messages.ts

@@ -11,10 +11,6 @@ export const listByChat = query({
 			chatId: v.id('chats'),
 			role: v.union(v.literal('user'), v.literal('assistant')),
 			content: v.string(),
-			imageBase64: v.optional(v.string()),
-			imageMediaType: v.optional(v.string()),
-			imagesBase64: v.optional(v.array(v.string())),
-			imagesMediaTypes: v.optional(v.array(v.string())),
 			followUpOptions: v.optional(v.array(v.string())),
 			source: v.union(v.literal('telegram'), v.literal('web')),
 			createdAt: v.number(),
@@ -22,11 +18,23 @@ export const listByChat = query({
 		})
 	),
 	handler: async (ctx, args) => {
-		return await ctx.db
+		const messages = await ctx.db
 			.query('messages')
 			.withIndex('by_chat_id_and_created_at', (q) => q.eq('chatId', args.chatId))
 			.order('asc')
 			.collect();
+
+		return messages.map((m) => ({
+			_id: m._id,
+			_creationTime: m._creationTime,
+			chatId: m.chatId,
+			role: m.role,
+			content: m.content,
+			followUpOptions: m.followUpOptions,
+			source: m.source,
+			createdAt: m.createdAt,
+			isStreaming: m.isStreaming
+		}));
 	}
 });
 

frontend/src/lib/convex/photoDrafts.ts

@@ -12,7 +12,6 @@ export const get = query({
 		photos: v.array(
 			v.object({
 				_id: v.id('photoDrafts'),
-				base64: v.string(),
 				mediaType: v.string()
 			})
 		)
@@ -28,7 +27,6 @@ export const get = query({
 		return {
 			photos: drafts.map((d) => ({
 				_id: d._id,
-				base64: d.base64,
 				mediaType: d.mediaType
 			}))
 		};

frontend/src/lib/convex/photoRequests.ts

@@ -21,6 +21,20 @@ const photoRequestValidator = v.object({
 	createdAt: v.number()
 });
 
+const photoRequestLightValidator = v.object({
+	_id: v.id('photoRequests'),
+	status: v.union(
+		v.literal('pending'),
+		v.literal('countdown'),
+		v.literal('capture_now'),
+		v.literal('captured'),
+		v.literal('accepted'),
+		v.literal('rejected')
+	),
+	photoMediaType: v.optional(v.string()),
+	thumbnailBase64: v.optional(v.string())
+});
+
 export const create = mutation({
 	args: {
 		chatId: v.id('chats'),
@@ -29,6 +43,17 @@ export const create = mutation({
 	},
 	returns: v.id('photoRequests'),
 	handler: async (ctx, args) => {
+		const oldRequests = await ctx.db
+			.query('photoRequests')
+			.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
+			.take(20);
+
+		for (const req of oldRequests) {
+			if (req.status === 'pending' || req.status === 'countdown' || req.status === 'capture_now') {
+				await ctx.db.patch(req._id, { status: 'rejected' });
+			}
+		}
+
 		return await ctx.db.insert('photoRequests', {
 			chatId: args.chatId,
 			requesterId: args.requesterId,
@@ -55,15 +80,20 @@ export const submitPhoto = mutation({
 		photoMediaType: v.string(),
 		thumbnailBase64: v.optional(v.string())
 	},
-	returns: v.null(),
+	returns: v.boolean(),
 	handler: async (ctx, args) => {
+		const req = await ctx.db.get(args.requestId);
+		if (!req || req.status !== 'capture_now') {
+			return false;
+		}
+
 		await ctx.db.patch(args.requestId, {
 			status: 'captured',
 			photoBase64: args.photoBase64,
 			photoMediaType: args.photoMediaType,
 			thumbnailBase64: args.thumbnailBase64
 		});
-		return null;
+		return true;
 	}
 });
 
@@ -78,24 +108,39 @@ export const markAccepted = mutation({
 
 export const markRejected = mutation({
 	args: { requestId: v.id('photoRequests') },
-	returns: v.null(),
+	returns: v.boolean(),
 	handler: async (ctx, args) => {
-		await ctx.db.patch(args.requestId, { status: 'rejected' });
-		return null;
+		const req = await ctx.db.get(args.requestId);
+		if (!req || req.status === 'accepted' || req.status === 'rejected') {
+			return false;
 		}
+		await ctx.db.patch(req._id, { status: 'rejected' });
+		return true;
+	}
+});
+
+const captureNowLightValidator = v.object({
+	_id: v.id('photoRequests'),
+	status: v.literal('capture_now')
 });
 
 export const getCaptureNowRequest = query({
 	args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) },
-	returns: v.union(photoRequestValidator, v.null()),
+	returns: v.union(captureNowLightValidator, v.null()),
 	handler: async (ctx, args) => {
+		const now = Date.now();
+		const maxAge = 60 * 1000;
+
 		const requests = await ctx.db
 			.query('photoRequests')
 			.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
 			.order('desc')
 			.take(50);
 
-		return requests.find((r) => r.status === 'capture_now') ?? null;
+		const found = requests.find((r) => r.status === 'capture_now' && now - r.createdAt < maxAge);
+		if (!found) return null;
+
+		return { _id: found._id, status: 'capture_now' as const };
 	}
 });
 
@@ -115,22 +160,99 @@ export const getActiveForCapture = query({
 
 export const getMyActiveRequest = query({
 	args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) },
-	returns: v.union(photoRequestValidator, v.null()),
+	returns: v.union(photoRequestLightValidator, v.null()),
 	handler: async (ctx, args) => {
 		const requests = await ctx.db
 			.query('photoRequests')
 			.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
+			.order('desc')
 			.take(100);
 
 		if (!args.deviceId) return null;
 
-		return (
-			requests.find(
+		const found = requests.find(
 			(r) =>
 				r.requesterId === args.deviceId &&
 				(r.status === 'countdown' || r.status === 'capture_now' || r.status === 'captured')
-			) ?? null
 		);
+
+		if (!found) return null;
+
+		return {
+			_id: found._id,
+			status: found.status,
+			photoMediaType: found.photoMediaType,
+			thumbnailBase64: found.thumbnailBase64
+		};
+	}
+});
+
+export const getPhotoData = query({
+	args: { requestId: v.id('photoRequests') },
+	returns: v.union(
+		v.object({
+			photoBase64: v.string(),
+			photoMediaType: v.string(),
+			thumbnailBase64: v.optional(v.string())
+		}),
+		v.null()
+	),
+	handler: async (ctx, args) => {
+		const req = await ctx.db.get(args.requestId);
+		if (!req || !req.photoBase64 || !req.photoMediaType) return null;
+
+		return {
+			photoBase64: req.photoBase64,
+			photoMediaType: req.photoMediaType,
+			thumbnailBase64: req.thumbnailBase64
+		};
+	}
+});
+
+export const getPhotoPreview = query({
+	args: { requestId: v.id('photoRequests') },
+	returns: v.union(
+		v.object({
+			thumbnailBase64: v.string(),
+			photoMediaType: v.string()
+		}),
+		v.null()
+	),
+	handler: async (ctx, args) => {
+		const req = await ctx.db.get(args.requestId);
+		if (!req || !req.photoMediaType) return null;
+
+		return {
+			thumbnailBase64: req.thumbnailBase64 || req.photoBase64 || '',
+			photoMediaType: req.photoMediaType
+		};
+	}
+});
+
+export const acceptPhotoToDraft = mutation({
+	args: {
+		requestId: v.id('photoRequests'),
+		chatId: v.id('chats'),
+		deviceId: v.string()
+	},
+	returns: v.id('photoDrafts'),
+	handler: async (ctx, args) => {
+		const req = await ctx.db.get(args.requestId);
+		if (!req || !req.photoBase64 || !req.photoMediaType) {
+			throw new Error('Photo request not found or has no photo');
+		}
+
+		const draftId = await ctx.db.insert('photoDrafts', {
+			chatId: args.chatId,
+			deviceId: args.deviceId,
+			base64: req.photoBase64,
+			mediaType: req.photoMediaType,
+			createdAt: Date.now()
+		});
+
+		await ctx.db.patch(args.requestId, { status: 'accepted' });
+
+		return draftId;
 	}
 });
 

frontend/src/routes/[mnemonic]/+page.svelte

@@ -4,7 +4,11 @@
 	import { getContext, onMount } from 'svelte';
 	import { SvelteSet } from 'svelte/reactivity';
 	import { useQuery, useConvexClient } from 'convex-svelte';
-	import { usePollingQuery, usePollingMutation } from '$lib/convex-polling.svelte';
+	import {
+		usePollingQuery,
+		usePollingMutation,
+		usePollingClient
+	} from '$lib/convex-polling.svelte';
 	import { api } from '$lib/convex/_generated/api';
 	import type { Id } from '$lib/convex/_generated/dataModel';
 	import ChatMessage from '$lib/components/ChatMessage.svelte';
@@ -30,7 +34,6 @@
 	let activeRequestId: Id<'photoRequests'> | null = $state(null);
 	let previewPhoto: {
 		thumbnail: string;
-		fullBase64: string;
 		mediaType: string;
 		requestId: Id<'photoRequests'>;
 	} | null = $state(null);
@@ -206,17 +209,24 @@
 
 	$effect(() => {
 		const req = myActiveRequest.data;
-		if (req?.status === 'captured' && req.photoBase64 && req.photoMediaType) {
+		if (req?.status === 'captured' && req.photoMediaType) {
 			if (shownPreviewIds.has(req._id)) return;
 			shownPreviewIds.add(req._id);
+
+			const client = pollingClient ?? clientWs;
+			if (client) {
+				client.query(api.photoRequests.getPhotoPreview, { requestId: req._id }).then((data) => {
+					if (data) {
 						previewPhoto = {
-				thumbnail: req.thumbnailBase64 || req.photoBase64,
-				fullBase64: req.photoBase64,
-				mediaType: req.photoMediaType,
+							thumbnail: data.thumbnailBase64,
+							mediaType: data.photoMediaType,
 							requestId: req._id
 						};
 					}
 				});
+			}
+		}
+	});
 
 	let prevMessageCount = 0;
 	let prevLastMessageId: string | undefined;
@@ -232,6 +242,7 @@
 	});
 
 	const clientWs = usePolling ? null : useConvexClient();
+	const pollingClient = usePolling ? usePollingClient() : null;
 	const createMessagePoll = usePolling ? usePollingMutation(api.messages.create) : null;
 	const registerDevicePoll = usePolling ? usePollingMutation(api.devicePairings.register) : null;
 	const heartbeatPoll = usePolling ? usePollingMutation(api.devicePairings.heartbeat) : null;
@@ -246,8 +257,10 @@
 		? usePollingMutation(api.photoRequests.markCaptureNow)
 		: null;
 	const submitPhotoPoll = usePolling ? usePollingMutation(api.photoRequests.submitPhoto) : null;
-	const markAcceptedPoll = usePolling ? usePollingMutation(api.photoRequests.markAccepted) : null;
 	const markRejectedPoll = usePolling ? usePollingMutation(api.photoRequests.markRejected) : null;
+	const acceptPhotoToDraftPoll = usePolling
+		? usePollingMutation(api.photoRequests.acceptPhotoToDraft)
+		: null;
 
 	$effect(() => {
 		if (!chatId || !deviceId) return;
@@ -467,21 +480,16 @@
 	function handlePreviewAccept() {
 		if (!previewPhoto || !chatId) return;
 
-		const photo = { base64: previewPhoto.fullBase64, mediaType: previewPhoto.mediaType };
 		const reqId = previewPhoto.requestId;
 		previewPhoto = null;
 
-		if (usePolling && addPhotoPoll && markAcceptedPoll) {
-			addPhotoPoll({ chatId, deviceId, photo });
-			markAcceptedPoll({ requestId: reqId });
+		if (usePolling && acceptPhotoToDraftPoll) {
+			acceptPhotoToDraftPoll({ requestId: reqId, chatId, deviceId });
 		} else if (clientWs) {
-			clientWs.mutation(api.photoDrafts.addPhoto, {
+			clientWs.mutation(api.photoRequests.acceptPhotoToDraft, {
+				requestId: reqId,
 				chatId,
-				deviceId,
-				photo
-			});
-			clientWs.mutation(api.photoRequests.markAccepted, {
-				requestId: reqId
+				deviceId
 			});
 		}
 	}
@@ -661,6 +669,10 @@
 	{/if}
 
 	{#if hasCamera && isPaired}
-		<SilentCapture bind:this={silentCaptureRef} oncapture={handleSilentCapture} />
+		<SilentCapture
+			bind:this={silentCaptureRef}
+			oncapture={handleSilentCapture}
+			onunpair={handleUnpair}
+		/>
 	{/if}
 </div>

frontend/vite.config.ts

@@ -4,5 +4,5 @@ import { defineConfig } from 'vite';
 
 export default defineConfig({
 	plugins: [tailwindcss(), sveltekit()],
-	server: { allowedHosts: ['porter-davidson-fibre-handhelds.trycloudflare.com'] }
+	server: { allowedHosts: ['reasonable-duncan-stations-parking.trycloudflare.com'] }
 });