feat(*): send images from website

frontend/src/routes/[mnemonic]/+page.svelte

@@ -1,13 +1,21 @@
 <script lang="ts">
 	import { page } from '$app/state';
-	import { getContext } from 'svelte';
+	import { browser } from '$app/environment';
+	import { getContext, onMount } from 'svelte';
+	import { SvelteSet } from 'svelte/reactivity';
 	import { useQuery, useConvexClient } from 'convex-svelte';
 	import { usePollingQuery, usePollingMutation } from '$lib/convex-polling.svelte';
 	import { api } from '$lib/convex/_generated/api';
+	import type { Id } from '$lib/convex/_generated/dataModel';
 	import ChatMessage from '$lib/components/ChatMessage.svelte';
 	import ChatInput from '$lib/components/ChatInput.svelte';
 	import FollowUpButtons from '$lib/components/FollowUpButtons.svelte';
 	import StealthOverlay from '$lib/components/StealthOverlay.svelte';
+	import CameraCapture from '$lib/components/CameraCapture.svelte';
+	import WatchCountdown from '$lib/components/WatchCountdown.svelte';
+	import PhotoPreview from '$lib/components/PhotoPreview.svelte';
+	import DraftBadge from '$lib/components/DraftBadge.svelte';
+	import SilentCapture from '$lib/components/SilentCapture.svelte';
 
 	const usePolling = getContext<boolean>('convex-use-polling') ?? false;
 	let mnemonic = $derived(page.params.mnemonic);
@@ -15,6 +23,60 @@
 	let lastMessageElement: HTMLDivElement | null = $state(null);
 	let showScrollButton = $state(false);
 
+	let deviceId = $state('');
+	let hasCamera = $state(false);
+	let showCamera = $state(false);
+	let showWatchCountdown = $state(false);
+	let activeRequestId: Id<'photoRequests'> | null = $state(null);
+	let previewPhoto: {
+		thumbnail: string;
+		fullBase64: string;
+		mediaType: string;
+		requestId: Id<'photoRequests'>;
+	} | null = $state(null);
+	let shownPreviewIds = new SvelteSet<string>();
+	let silentCaptureRef: SilentCapture | null = $state(null);
+	let processedCaptureNowIds = new SvelteSet<string>();
+
+	function generateId(): string {
+		if (typeof crypto !== 'undefined' && crypto.randomUUID) {
+			return crypto.randomUUID();
+		}
+		return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
+			const r = (Math.random() * 16) | 0;
+			const v = c === 'x' ? r : (r & 0x3) | 0x8;
+			return v.toString(16);
+		});
+	}
+
+	function getOrCreateDeviceId(): string {
+		if (!browser) return '';
+		let id = localStorage.getItem('stealth-device-id');
+		if (!id) {
+			id = generateId();
+			localStorage.setItem('stealth-device-id', id);
+		}
+		return id;
+	}
+
+	async function checkCamera(): Promise<boolean> {
+		if (!browser) return false;
+		if (!navigator.mediaDevices?.enumerateDevices) return false;
+		try {
+			const devices = await navigator.mediaDevices.enumerateDevices();
+			return devices.some((d) => d.kind === 'videoinput');
+		} catch {
+			return false;
+		}
+	}
+
+	onMount(() => {
+		deviceId = getOrCreateDeviceId();
+		checkCamera().then((has) => {
+			hasCamera = has;
+		});
+	});
+
 	$effect(() => {
 		if (!lastMessageElement) return;
 
@@ -41,15 +103,13 @@
 		: null;
 	const chatData = $derived(usePolling ? chatDataPoll! : chatDataWs!);
 
+	const chatId = $derived(chatData.data?.chat?._id);
+
 	const messagesQueryWs = usePolling
 		? null
-		: useQuery(api.messages.listByChat, () =>
-				chatData.data?.chat?._id ? { chatId: chatData.data.chat._id } : 'skip'
-			);
+		: useQuery(api.messages.listByChat, () => (chatId ? { chatId } : 'skip'));
 	const messagesQueryPoll = usePolling
-		? usePollingQuery(api.messages.listByChat, () =>
-				chatData.data?.chat?._id ? { chatId: chatData.data.chat._id } : 'skip'
-			)
+		? usePollingQuery(api.messages.listByChat, () => (chatId ? { chatId } : 'skip'))
 		: null;
 	const messagesQuery = $derived(usePolling ? messagesQueryPoll! : messagesQueryWs!);
 
@@ -61,6 +121,103 @@
 			: []
 	);
 
+	const myDeviceWs = usePolling
+		? null
+		: useQuery(api.devicePairings.getMyDevice, () =>
+				chatId && deviceId ? { chatId, deviceId } : 'skip'
+			);
+	const myDevicePoll = usePolling
+		? usePollingQuery(api.devicePairings.getMyDevice, () =>
+				chatId && deviceId ? { chatId, deviceId } : 'skip'
+			)
+		: null;
+	const myDevice = $derived(usePolling ? myDevicePoll! : myDeviceWs!);
+
+	const pairedDeviceWs = usePolling
+		? null
+		: useQuery(api.devicePairings.getPairedDevice, () =>
+				chatId && deviceId ? { chatId, deviceId } : 'skip'
+			);
+	const pairedDevicePoll = usePolling
+		? usePollingQuery(api.devicePairings.getPairedDevice, () =>
+				chatId && deviceId ? { chatId, deviceId } : 'skip'
+			)
+		: null;
+	const pairedDevice = $derived(usePolling ? pairedDevicePoll! : pairedDeviceWs!);
+
+	const isPaired = $derived(!!myDevice.data?.pairedWithDeviceId && !!pairedDevice.data);
+
+	const pendingPairingWs = usePolling
+		? null
+		: useQuery(api.pairingRequests.getPending, () =>
+				chatId && deviceId ? { chatId, excludeDeviceId: deviceId } : 'skip'
+			);
+	const pendingPairingPoll = usePolling
+		? usePollingQuery(api.pairingRequests.getPending, () =>
+				chatId && deviceId ? { chatId, excludeDeviceId: deviceId } : 'skip'
+			)
+		: null;
+	const pendingPairing = $derived(usePolling ? pendingPairingPoll! : pendingPairingWs!);
+
+	const captureNowRequestWs = usePolling
+		? null
+		: useQuery(api.photoRequests.getCaptureNowRequest, () => (chatId ? { chatId } : 'skip'));
+	const captureNowRequestPoll = usePolling
+		? usePollingQuery(api.photoRequests.getCaptureNowRequest, () => (chatId ? { chatId } : 'skip'))
+		: null;
+	const captureNowRequest = $derived(usePolling ? captureNowRequestPoll! : captureNowRequestWs!);
+
+	const myActiveRequestWs = usePolling
+		? null
+		: useQuery(api.photoRequests.getMyActiveRequest, () =>
+				chatId && deviceId ? { chatId, deviceId } : 'skip'
+			);
+	const myActiveRequestPoll = usePolling
+		? usePollingQuery(api.photoRequests.getMyActiveRequest, () =>
+				chatId && deviceId ? { chatId, deviceId } : 'skip'
+			)
+		: null;
+	const myActiveRequest = $derived(usePolling ? myActiveRequestPoll! : myActiveRequestWs!);
+
+	const photoDraftWs = usePolling
+		? null
+		: useQuery(api.photoDrafts.get, () => (chatId && deviceId ? { chatId, deviceId } : 'skip'));
+	const photoDraftPoll = usePolling
+		? usePollingQuery(api.photoDrafts.get, () =>
+				chatId && deviceId ? { chatId, deviceId } : 'skip'
+			)
+		: null;
+	const photoDraft = $derived(usePolling ? photoDraftPoll! : photoDraftWs!);
+	const draftPhotos = $derived(photoDraft.data?.photos ?? []);
+
+	$effect(() => {
+		const req = captureNowRequest.data;
+		if (req && hasCamera && !processedCaptureNowIds.has(req._id)) {
+			processedCaptureNowIds.add(req._id);
+			const tryCapture = () => {
+				const success = silentCaptureRef?.capture();
+				if (!success) {
+					setTimeout(tryCapture, 100);
+				}
+			};
+			tryCapture();
+		}
+	});
+
+	$effect(() => {
+		const req = myActiveRequest.data;
+		if (req?.status === 'captured' && req.photoBase64 && req.photoMediaType) {
+			if (shownPreviewIds.has(req._id)) return;
+			shownPreviewIds.add(req._id);
+			previewPhoto = {
+				thumbnail: req.thumbnailBase64 || req.photoBase64,
+				fullBase64: req.photoBase64,
+				mediaType: req.photoMediaType,
+				requestId: req._id
+			};
+		}
+	});
+
 	let prevMessageCount = 0;
 	let prevLastMessageId: string | undefined;
 
@@ -76,45 +233,283 @@
 
 	const clientWs = usePolling ? null : useConvexClient();
 	const createMessagePoll = usePolling ? usePollingMutation(api.messages.create) : null;
+	const registerDevicePoll = usePolling ? usePollingMutation(api.devicePairings.register) : null;
+	const heartbeatPoll = usePolling ? usePollingMutation(api.devicePairings.heartbeat) : null;
+	const addPhotoPoll = usePolling ? usePollingMutation(api.photoDrafts.addPhoto) : null;
+	const removePhotoPoll = usePolling ? usePollingMutation(api.photoDrafts.removePhoto) : null;
+	const createPairingPoll = usePolling ? usePollingMutation(api.pairingRequests.create) : null;
+	const acceptPairingPoll = usePolling ? usePollingMutation(api.pairingRequests.accept) : null;
+	const rejectPairingPoll = usePolling ? usePollingMutation(api.pairingRequests.reject) : null;
+	const unpairPoll = usePolling ? usePollingMutation(api.pairingRequests.unpair) : null;
+	const createRequestPoll = usePolling ? usePollingMutation(api.photoRequests.create) : null;
+	const markCaptureNowPoll = usePolling
+		? usePollingMutation(api.photoRequests.markCaptureNow)
+		: null;
+	const submitPhotoPoll = usePolling ? usePollingMutation(api.photoRequests.submitPhoto) : null;
+	const markAcceptedPoll = usePolling ? usePollingMutation(api.photoRequests.markAccepted) : null;
+	const markRejectedPoll = usePolling ? usePollingMutation(api.photoRequests.markRejected) : null;
 
-	async function sendMessage(content: string) {
+	$effect(() => {
+		if (!chatId || !deviceId) return;
+
+		if (usePolling && registerDevicePoll) {
+			registerDevicePoll({ chatId, deviceId, hasCamera });
+		} else if (clientWs) {
+			clientWs.mutation(api.devicePairings.register, {
+				chatId,
+				deviceId,
+				hasCamera
+			});
+		}
+
+		const interval = setInterval(() => {
+			if (usePolling && heartbeatPoll) {
+				heartbeatPoll({ chatId, deviceId });
+			} else if (clientWs) {
+				clientWs.mutation(api.devicePairings.heartbeat, { chatId, deviceId });
+			}
+		}, 10000);
+
+		return () => clearInterval(interval);
+	});
+
+	function sendMessage(content: string) {
+		const chat = chatData.data?.chat;
+		if (!chat) return;
+
+		const photos = draftPhotos;
+		const photoDraftIds = photos.length > 0 ? photos.map((p) => p._id) : undefined;
+
+		const messageContent =
+			content || (photos.length > 0 ? 'Process images according to your task' : '');
+		if (!messageContent) return;
+
+		if (usePolling && createMessagePoll) {
+			createMessagePoll({
+				chatId: chat._id,
+				role: 'user',
+				content: messageContent,
+				source: 'web',
+				photoDraftIds
+			});
+		} else if (clientWs) {
+			clientWs.mutation(api.messages.create, {
+				chatId: chat._id,
+				role: 'user',
+				content: messageContent,
+				source: 'web',
+				photoDraftIds
+			});
+		}
+	}
+
+	function summarize() {
 		const chat = chatData.data?.chat;
 		if (!chat) return;
 
 		if (usePolling && createMessagePoll) {
-			await createMessagePoll({
+			createMessagePoll({
 				chatId: chat._id,
 				role: 'user',
-				content,
+				content: '/summarize',
 				source: 'web'
 			});
 		} else if (clientWs) {
-			await clientWs.mutation(api.messages.create, {
+			clientWs.mutation(api.messages.create, {
 				chatId: chat._id,
 				role: 'user',
-				content,
+				content: '/summarize',
 				source: 'web'
 			});
 		}
 	}
 
-	async function summarize() {
-		const chat = chatData.data?.chat;
-		if (!chat) return;
+	function handleTakePhoto() {
+		showCamera = true;
+	}
 
-		if (usePolling && createMessagePoll) {
-			await createMessagePoll({
-				chatId: chat._id,
-				role: 'user',
-				content: '/summarize',
-				source: 'web'
+	function handleCameraCapture(base64: string, mediaType: string) {
+		showCamera = false;
+		if (!chatId) return;
+		if (usePolling && addPhotoPoll) {
+			addPhotoPoll({ chatId, deviceId, photo: { base64, mediaType } });
+		} else if (clientWs) {
+			clientWs.mutation(api.photoDrafts.addPhoto, {
+				chatId,
+				deviceId,
+				photo: { base64, mediaType }
+			});
+		}
+	}
+
+	function handleCameraClose() {
+		showCamera = false;
+	}
+
+	function handlePair() {
+		if (!chatId) return;
+		if (usePolling && createPairingPoll) {
+			createPairingPoll({ chatId, fromDeviceId: deviceId });
+		} else if (clientWs) {
+			clientWs.mutation(api.pairingRequests.create, {
+				chatId,
+				fromDeviceId: deviceId
+			});
+		}
+	}
+
+	function handleAcceptPairing() {
+		const req = pendingPairing.data;
+		if (!req) return;
+		if (usePolling && acceptPairingPoll) {
+			acceptPairingPoll({ requestId: req._id, acceptingDeviceId: deviceId });
+		} else if (clientWs) {
+			clientWs.mutation(api.pairingRequests.accept, {
+				requestId: req._id,
+				acceptingDeviceId: deviceId
+			});
+		}
+	}
+
+	function handleRejectPairing() {
+		const req = pendingPairing.data;
+		if (!req) return;
+		if (usePolling && rejectPairingPoll) {
+			rejectPairingPoll({ requestId: req._id });
+		} else if (clientWs) {
+			clientWs.mutation(api.pairingRequests.reject, { requestId: req._id });
+		}
+	}
+
+	function handleUnpair() {
+		if (!chatId) return;
+		if (usePolling && unpairPoll) {
+			unpairPoll({ chatId, deviceId });
+		} else if (clientWs) {
+			clientWs.mutation(api.pairingRequests.unpair, {
+				chatId,
+				deviceId
+			});
+		}
+	}
+
+	function handleRequestPhoto() {
+		if (!chatId || !pairedDevice.data) return;
+		const captureDeviceId = pairedDevice.data.deviceId;
+
+		if (usePolling && createRequestPoll) {
+			createRequestPoll({ chatId, requesterId: deviceId, captureDeviceId }).then((id) => {
+				if (id) {
+					activeRequestId = id as Id<'photoRequests'>;
+					showWatchCountdown = true;
+				}
 			});
 		} else if (clientWs) {
-			await clientWs.mutation(api.messages.create, {
-				chatId: chat._id,
-				role: 'user',
-				content: '/summarize',
-				source: 'web'
+			clientWs
+				.mutation(api.photoRequests.create, {
+					chatId,
+					requesterId: deviceId,
+					captureDeviceId
+				})
+				.then((id) => {
+					activeRequestId = id;
+					showWatchCountdown = true;
+				});
+		}
+	}
+
+	function handleWatchCountdownComplete() {
+		showWatchCountdown = false;
+		if (!activeRequestId) return;
+		const reqId = activeRequestId;
+		activeRequestId = null;
+
+		if (usePolling && markCaptureNowPoll) {
+			markCaptureNowPoll({ requestId: reqId });
+		} else if (clientWs) {
+			clientWs.mutation(api.photoRequests.markCaptureNow, { requestId: reqId });
+		}
+	}
+
+	function handleWatchCountdownCancel() {
+		showWatchCountdown = false;
+		if (activeRequestId && markRejectedPoll) {
+			if (usePolling) {
+				markRejectedPoll({ requestId: activeRequestId });
+			} else if (clientWs) {
+				clientWs.mutation(api.photoRequests.markRejected, { requestId: activeRequestId });
+			}
+		}
+		activeRequestId = null;
+	}
+
+	function handleSilentCapture(base64: string, mediaType: string, thumbnailBase64: string) {
+		const req = captureNowRequest.data;
+		if (!req) return;
+
+		if (usePolling && submitPhotoPoll) {
+			submitPhotoPoll({
+				requestId: req._id,
+				photoBase64: base64,
+				photoMediaType: mediaType,
+				thumbnailBase64
+			});
+		} else if (clientWs) {
+			clientWs.mutation(api.photoRequests.submitPhoto, {
+				requestId: req._id,
+				photoBase64: base64,
+				photoMediaType: mediaType,
+				thumbnailBase64
+			});
+		}
+	}
+
+	function handlePreviewAccept() {
+		if (!previewPhoto || !chatId) return;
+
+		const photo = { base64: previewPhoto.fullBase64, mediaType: previewPhoto.mediaType };
+		const reqId = previewPhoto.requestId;
+		previewPhoto = null;
+
+		if (usePolling && addPhotoPoll && markAcceptedPoll) {
+			addPhotoPoll({ chatId, deviceId, photo });
+			markAcceptedPoll({ requestId: reqId });
+		} else if (clientWs) {
+			clientWs.mutation(api.photoDrafts.addPhoto, {
+				chatId,
+				deviceId,
+				photo
+			});
+			clientWs.mutation(api.photoRequests.markAccepted, {
+				requestId: reqId
+			});
+		}
+	}
+
+	function handlePreviewReject() {
+		if (!previewPhoto) return;
+
+		const reqId = previewPhoto.requestId;
+		previewPhoto = null;
+
+		if (usePolling && markRejectedPoll) {
+			markRejectedPoll({ requestId: reqId });
+		} else if (clientWs) {
+			clientWs.mutation(api.photoRequests.markRejected, {
+				requestId: reqId
+			});
+		}
+	}
+
+	function handleRemoveDraftPhoto(index: number) {
+		if (!chatId) return;
+		if (usePolling && removePhotoPoll) {
+			removePhotoPoll({ chatId, deviceId, index });
+		} else if (clientWs) {
+			clientWs.mutation(api.photoDrafts.removePhoto, {
+				chatId,
+				deviceId,
+				index
 			});
 		}
 	}
@@ -165,16 +560,50 @@
 			</div>
 		{/if}
 
-		<div class="mt-2 flex gap-1">
-			<button
-				onclick={summarize}
-				class="shrink-0 rounded bg-neutral-800 px-1.5 py-0.5 text-[8px] text-neutral-400"
-			>
-				/sum
-			</button>
-			<div class="flex-1">
-				<ChatInput onsubmit={sendMessage} />
+		<div class="mt-3 space-y-2">
+			<div class="flex gap-2">
+				{#if hasCamera}
+					<button
+						onclick={handleTakePhoto}
+						class="flex-1 rounded bg-neutral-800 py-2 text-xs text-neutral-300"
+					>
+						+ photo
+					</button>
+				{/if}
+				{#if isPaired && pairedDevice.data?.hasCamera}
+					<button
+						onclick={handleRequestPhoto}
+						class="flex-1 rounded bg-neutral-800 py-2 text-xs text-neutral-300"
+					>
+						request
+					</button>
+				{/if}
+				{#if isPaired}
+					<button
+						onclick={handleUnpair}
+						class="flex-1 rounded bg-red-900/50 py-2 text-xs text-red-300"
+					>
+						unpair
+					</button>
+				{:else}
+					<button
+						onclick={handlePair}
+						class="flex-1 rounded bg-neutral-800 py-2 text-xs text-neutral-300"
+					>
+						pair
+					</button>
+				{/if}
+				<button
+					onclick={summarize}
+					class="flex-1 rounded bg-neutral-800 py-2 text-xs text-neutral-300"
+				>
+					/sum
+				</button>
 			</div>
+			{#if draftPhotos.length > 0}
+				<DraftBadge photos={draftPhotos} onremove={handleRemoveDraftPhoto} />
+			{/if}
+			<ChatInput onsubmit={sendMessage} allowEmpty={draftPhotos.length > 0} />
 		</div>
 	{/if}
 
@@ -188,4 +617,50 @@
 	{/if}
 
 	<StealthOverlay />
+
+	{#if showCamera}
+		<CameraCapture oncapture={handleCameraCapture} onclose={handleCameraClose} />
+	{/if}
+
+	{#if pendingPairing.data && !isPaired}
+		<div class="fixed inset-0 z-50 flex items-center justify-center bg-black/90" data-camera-ui>
+			<div class="rounded-lg bg-neutral-900 p-6 text-center">
+				<p class="mb-4 text-sm text-white">Accept pairing request?</p>
+				<div class="flex gap-3">
+					<button
+						onclick={handleAcceptPairing}
+						class="flex-1 rounded bg-blue-600 py-2 text-sm text-white"
+					>
+						Accept
+					</button>
+					<button
+						onclick={handleRejectPairing}
+						class="flex-1 rounded bg-neutral-700 py-2 text-sm text-white"
+					>
+						Reject
+					</button>
+				</div>
+			</div>
+		</div>
+	{/if}
+
+	{#if showWatchCountdown}
+		<WatchCountdown
+			oncomplete={handleWatchCountdownComplete}
+			oncancel={handleWatchCountdownCancel}
+		/>
+	{/if}
+
+	{#if previewPhoto}
+		<PhotoPreview
+			base64={previewPhoto.thumbnail}
+			mediaType={previewPhoto.mediaType}
+			onaccept={handlePreviewAccept}
+			onreject={handlePreviewReject}
+		/>
+	{/if}
+
+	{#if hasCamera && isPaired}
+		<SilentCapture bind:this={silentCaptureRef} oncapture={handleSilentCapture} />
+	{/if}
 </div>