feat(*): send images from website

This commit is contained in:
h
2026-01-23 02:16:46 +01:00
parent 9f4dd8313e
commit b9c4296ca3
30 changed files with 1917 additions and 98 deletions

View File

@@ -9,6 +9,8 @@ from typing import Any
from aiogram import BaseMiddleware, Bot, F, Router, html, types
from aiogram.enums import ChatAction
from aiogram.types import (
BufferedInputFile,
InputMediaPhoto,
KeyboardButton,
ReplyKeyboardMarkup,
ReplyKeyboardRemove,
@@ -169,8 +171,13 @@ async def send_long_message(
)
async def process_message_from_web( # noqa: C901, PLR0912, PLR0915
convex_user_id: str, text: str, bot: Bot, convex_chat_id: str
async def process_message_from_web( # noqa: C901, PLR0912, PLR0913, PLR0915
convex_user_id: str,
text: str,
bot: Bot,
convex_chat_id: str,
images_base64: list[str] | None = None,
images_media_types: list[str] | None = None,
) -> None:
user = await convex.query("users:getById", {"userId": convex_user_id})
@@ -181,9 +188,32 @@ async def process_message_from_web( # noqa: C901, PLR0912, PLR0915
is_summarize = text == "/summarize"
if tg_chat_id and not is_summarize:
await bot.send_message(
tg_chat_id, f"📱 {html.quote(text)}", reply_markup=ReplyKeyboardRemove()
)
if images_base64 and images_media_types:
if len(images_base64) == 1:
photo_bytes = base64.b64decode(images_base64[0])
await bot.send_photo(
tg_chat_id,
BufferedInputFile(photo_bytes, "photo.jpg"),
caption=f"📱 {text}" if text else "📱",
reply_markup=ReplyKeyboardRemove(),
)
else:
media = []
img_pairs = zip(images_base64, images_media_types, strict=True)
for i, (img_b64, _) in enumerate(img_pairs):
photo_bytes = base64.b64decode(img_b64)
caption = f"📱 {text}" if i == 0 and text else None
media.append(
InputMediaPhoto(
media=BufferedInputFile(photo_bytes, f"photo_{i}.jpg"),
caption=caption,
)
)
await bot.send_media_group(tg_chat_id, media)
else:
await bot.send_message(
tg_chat_id, f"📱 {html.quote(text)}", reply_markup=ReplyKeyboardRemove()
)
api_key = user["geminiApiKey"]
model_name = user.get("model", "gemini-3-pro-preview")

View File

@@ -51,6 +51,8 @@ async def handle_pending_generation(bot: Bot, item: dict, item_id: str) -> None:
text=item["userMessage"],
bot=bot,
convex_chat_id=item["chatId"],
images_base64=item.get("imagesBase64"),
images_media_types=item.get("imagesMediaTypes"),
)
except Exception as e: # noqa: BLE001
logger.error(f"Error processing {item_id}: {e}")

View File

@@ -5,5 +5,8 @@ yarn.lock
bun.lock
bun.lockb
# Convex generated files
src/lib/convex/_generated/
# Miscellaneous
/static/

View File

@@ -22,9 +22,11 @@ export default defineConfig(
languageOptions: { globals: { ...globals.browser, ...globals.node } },
rules: {
// typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects.
// see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
'no-undef': 'off'
'no-undef': 'off',
'@typescript-eslint/no-unused-vars': [
'error',
{ argsIgnorePattern: '^_', varsIgnorePattern: '^_' }
]
}
},
{

View File

@@ -0,0 +1,184 @@
<script lang="ts">
import { onMount } from 'svelte';
interface Props {
showPreview?: boolean;
oncapture: (base64: string, mediaType: string) => void;
onclose: () => void;
}
let { showPreview = true, oncapture, onclose }: Props = $props();
let videoElement: HTMLVideoElement | null = $state(null);
let stream: MediaStream | null = $state(null);
let capturedImage: { base64: string; mediaType: string } | null = $state(null);
let error: string | null = $state(null);
let closed = false;
async function findUltraWideCamera(): Promise<string | null> {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const videoDevices = devices.filter((d) => d.kind === 'videoinput');
const ultraWide = videoDevices.find(
(d) => d.label.toLowerCase().includes('ultra') && d.label.toLowerCase().includes('back')
);
return ultraWide?.deviceId ?? null;
} catch {
return null;
}
}
async function startCamera() {
if (closed) return;
if (!navigator.mediaDevices?.getUserMedia) {
error = 'Camera not supported (requires HTTPS)';
return;
}
try {
stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: { ideal: 'environment' } },
audio: false
});
const ultraWideId = await findUltraWideCamera();
if (ultraWideId) {
stream.getTracks().forEach((t) => t.stop());
stream = await navigator.mediaDevices.getUserMedia({
video: {
deviceId: { exact: ultraWideId },
width: { ideal: 4032 },
height: { ideal: 3024 }
},
audio: false
});
} else {
stream.getTracks().forEach((t) => t.stop());
stream = await navigator.mediaDevices.getUserMedia({
video: {
facingMode: { ideal: 'environment' },
width: { ideal: 4032 },
height: { ideal: 3024 }
},
audio: false
});
}
if (videoElement && !closed) {
videoElement.srcObject = stream;
}
} catch (e) {
error = e instanceof Error ? e.message : 'Camera access denied';
}
}
function stopCamera() {
if (stream) {
stream.getTracks().forEach((track) => track.stop());
stream = null;
}
}
function capture() {
if (!videoElement) return;
const canvas = document.createElement('canvas');
canvas.width = videoElement.videoWidth;
canvas.height = videoElement.videoHeight;
const ctx = canvas.getContext('2d');
if (!ctx) return;
ctx.drawImage(videoElement, 0, 0);
const maxSize = 1920;
const scale = Math.min(maxSize / canvas.width, maxSize / canvas.height, 1);
const outCanvas = document.createElement('canvas');
outCanvas.width = Math.round(canvas.width * scale);
outCanvas.height = Math.round(canvas.height * scale);
const outCtx = outCanvas.getContext('2d');
if (!outCtx) return;
outCtx.drawImage(canvas, 0, 0, outCanvas.width, outCanvas.height);
const base64 = outCanvas.toDataURL('image/jpeg', 0.65).split(',')[1];
const mediaType = 'image/jpeg';
stopCamera();
if (showPreview) {
capturedImage = { base64, mediaType };
} else {
oncapture(base64, mediaType);
}
}
function acceptCapture() {
if (capturedImage) {
oncapture(capturedImage.base64, capturedImage.mediaType);
}
}
function retake() {
capturedImage = null;
startCamera();
}
function close() {
closed = true;
stopCamera();
onclose();
}
onMount(() => {
startCamera();
return () => {
closed = true;
stopCamera();
};
});
</script>
<div class="fixed inset-0 z-50 flex flex-col bg-black" data-camera-ui>
{#if error}
<div class="flex flex-1 flex-col items-center justify-center p-4">
<p class="mb-4 text-center text-sm text-red-400">{error}</p>
<button onclick={close} class="rounded bg-neutral-700 px-4 py-2 text-sm text-white">
Close
</button>
</div>
{:else if capturedImage}
<div class="relative min-h-0 flex-1">
<button class="absolute inset-0" onclick={acceptCapture}>
<img
src="data:{capturedImage.mediaType};base64,{capturedImage.base64}"
alt="Captured"
class="h-full w-full object-contain"
/>
</button>
</div>
<div class="flex gap-4 p-4">
<button onclick={retake} class="flex-1 rounded bg-neutral-700 py-3 text-sm text-white">
Retake
</button>
<button onclick={acceptCapture} class="flex-1 rounded bg-blue-600 py-3 text-sm text-white">
Use
</button>
</div>
{:else}
<video bind:this={videoElement} autoplay playsinline muted class="h-full w-full object-cover"
></video>
<button
onclick={close}
class="absolute top-4 right-4 flex h-8 w-8 items-center justify-center rounded-full bg-black/50 text-white"
>
×
</button>
<button
onclick={capture}
aria-label="Capture photo"
class="absolute bottom-8 left-1/2 h-16 w-16 -translate-x-1/2 rounded-full border-4 border-white bg-white/20"
></button>
{/if}
</div>

View File

@@ -0,0 +1,144 @@
<script lang="ts">
import { onMount } from 'svelte';
interface Props {
oncomplete: (base64: string, mediaType: string, thumbnailBase64: string) => void;
oncancel: () => void;
}
let { oncomplete, oncancel }: Props = $props();
let count = $state(3);
let videoElement: HTMLVideoElement | null = $state(null);
let stream: MediaStream | null = $state(null);
let error: string | null = $state(null);
let cancelled = false;
let countdownInterval: ReturnType<typeof setInterval> | null = null;
async function startCamera() {
if (cancelled) return;
if (!navigator.mediaDevices?.getUserMedia) {
error = 'Camera not supported (requires HTTPS)';
return;
}
try {
const constraints: MediaStreamConstraints = {
video: {
facingMode: { ideal: 'environment' },
width: { ideal: 1920 },
height: { ideal: 1080 }
},
audio: false
};
stream = await navigator.mediaDevices.getUserMedia(constraints);
if (videoElement && !cancelled) {
videoElement.srcObject = stream;
startCountdown();
}
} catch (e) {
error = e instanceof Error ? e.message : 'Camera access denied';
}
}
function stopCamera() {
if (countdownInterval) {
clearInterval(countdownInterval);
countdownInterval = null;
}
if (stream) {
stream.getTracks().forEach((track) => track.stop());
stream = null;
}
}
function startCountdown() {
countdownInterval = setInterval(() => {
if (cancelled) {
stopCamera();
return;
}
count--;
if (count === 0) {
if (countdownInterval) clearInterval(countdownInterval);
capture();
}
}, 1000);
}
function capture() {
if (cancelled || !videoElement) {
stopCamera();
return;
}
const canvas = document.createElement('canvas');
canvas.width = videoElement.videoWidth;
canvas.height = videoElement.videoHeight;
const ctx = canvas.getContext('2d');
if (!ctx) {
stopCamera();
oncancel();
return;
}
ctx.drawImage(videoElement, 0, 0);
const base64 = canvas.toDataURL('image/jpeg', 0.85).split(',')[1];
const mediaType = 'image/jpeg';
const thumbMaxSize = 800;
const scale = Math.min(thumbMaxSize / canvas.width, thumbMaxSize / canvas.height, 1);
const thumbCanvas = document.createElement('canvas');
thumbCanvas.width = Math.round(canvas.width * scale);
thumbCanvas.height = Math.round(canvas.height * scale);
const thumbCtx = thumbCanvas.getContext('2d');
if (thumbCtx) {
thumbCtx.drawImage(canvas, 0, 0, thumbCanvas.width, thumbCanvas.height);
}
const thumbnailBase64 = thumbCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
stopCamera();
oncomplete(base64, mediaType, thumbnailBase64);
}
function handleCancel() {
cancelled = true;
stopCamera();
oncancel();
}
onMount(() => {
startCamera();
return () => {
cancelled = true;
stopCamera();
};
});
</script>
<div class="fixed inset-0 z-50 flex flex-col bg-black" data-camera-ui>
{#if error}
<div class="flex flex-1 flex-col items-center justify-center p-4">
<p class="mb-4 text-center text-sm text-red-400">{error}</p>
<button onclick={handleCancel} class="rounded bg-neutral-700 px-4 py-2 text-sm text-white">
Close
</button>
</div>
{:else}
<div class="relative flex-1">
<video bind:this={videoElement} autoplay playsinline muted class="h-full w-full object-cover"
></video>
<div class="absolute inset-0 flex items-center justify-center">
<span class="text-8xl font-bold text-white drop-shadow-lg">{count}</span>
</div>
</div>
<div class="p-4 text-center">
<button onclick={handleCancel} class="text-sm text-neutral-400">Cancel</button>
</div>
{/if}
</div>

View File

@@ -2,15 +2,16 @@
interface Props {
onsubmit: (message: string) => void;
disabled?: boolean;
allowEmpty?: boolean;
}
let { onsubmit, disabled = false }: Props = $props();
let { onsubmit, disabled = false, allowEmpty = false }: Props = $props();
let value = $state('');
function handleSubmit(e: Event) {
e.preventDefault();
const trimmed = value.trim();
if (trimmed && !disabled) {
if ((trimmed || allowEmpty) && !disabled) {
onsubmit(trimmed);
value = '';
}

View File

@@ -0,0 +1,27 @@
<script lang="ts">
interface Photo {
base64: string;
mediaType: string;
}
interface Props {
photos: Photo[];
onremove: (index: number) => void;
}
let { photos, onremove }: Props = $props();
</script>
{#if photos.length > 0}
<div class="flex flex-wrap gap-1">
{#each photos as _photo, i (i)}
<button
onclick={() => onremove(i)}
class="flex items-center gap-1 rounded bg-blue-600/30 px-1.5 py-0.5 text-[8px] text-blue-300"
>
<span>photo {i + 1}</span>
<span class="text-blue-400">&times;</span>
</button>
{/each}
</div>
{/if}

View File

@@ -0,0 +1,65 @@
<script lang="ts">
interface Props {
hasCamera: boolean;
hasOnlineDevices: boolean;
ontakephoto: () => void;
onrequestphoto: () => void;
}
let { hasCamera, hasOnlineDevices, ontakephoto, onrequestphoto }: Props = $props();
let menuOpen = $state(false);
function handleClick() {
if (hasCamera && hasOnlineDevices) {
menuOpen = !menuOpen;
} else if (hasOnlineDevices) {
onrequestphoto();
} else {
ontakephoto();
}
}
function handleTakePhoto() {
menuOpen = false;
ontakephoto();
}
function handleRequestPhoto() {
menuOpen = false;
onrequestphoto();
}
function handleBackdropClick() {
menuOpen = false;
}
</script>
<div class="relative">
<button
onclick={handleClick}
class="shrink-0 rounded bg-neutral-800 px-1.5 py-0.5 text-[8px] text-neutral-400"
>
+
</button>
{#if menuOpen}
<button class="fixed inset-0 z-40" onclick={handleBackdropClick} aria-label="Close menu"
></button>
<div
class="absolute bottom-full left-0 z-50 mb-1 overflow-hidden rounded bg-neutral-800 shadow-lg"
>
<button
onclick={handleTakePhoto}
class="block w-full px-3 py-2 text-left text-[10px] whitespace-nowrap text-white hover:bg-neutral-700"
>
Take photo
</button>
<button
onclick={handleRequestPhoto}
class="block w-full px-3 py-2 text-left text-[10px] whitespace-nowrap text-white hover:bg-neutral-700"
>
Request photo
</button>
</div>
{/if}
</div>

View File

@@ -0,0 +1,27 @@
<script lang="ts">
interface Props {
base64: string;
mediaType: string;
onaccept: () => void;
onreject: () => void;
}
let { base64, mediaType, onaccept, onreject }: Props = $props();
</script>
<div class="fixed inset-0 z-50 overflow-auto bg-black" data-camera-ui>
<button class="block min-h-full min-w-full" onclick={onaccept}>
<img
src="data:{mediaType};base64,{base64}"
alt="Preview"
class="min-h-dvh min-w-full object-cover"
/>
</button>
</div>
<button
onclick={onreject}
class="fixed top-4 right-4 z-[9999] flex h-10 w-10 items-center justify-center rounded-full bg-red-600 text-xl text-white shadow-lg"
data-camera-ui
>
×
</button>

View File

@@ -0,0 +1,22 @@
<script lang="ts">
interface Props {
onaccept: () => void;
ondecline: () => void;
}
let { onaccept, ondecline }: Props = $props();
</script>
<div class="fixed inset-0 z-50 flex items-center justify-center bg-black/80 p-4" data-camera-ui>
<div class="w-full max-w-xs rounded-lg bg-neutral-900 p-4">
<p class="mb-4 text-center text-sm text-white">Photo requested</p>
<div class="flex gap-3">
<button onclick={ondecline} class="flex-1 rounded bg-neutral-700 py-2 text-sm text-white">
Decline
</button>
<button onclick={onaccept} class="flex-1 rounded bg-blue-600 py-2 text-sm text-white">
Capture
</button>
</div>
</div>
</div>

View File

@@ -0,0 +1,126 @@
<script lang="ts">
import { onMount } from 'svelte';
interface Props {
oncapture: (base64: string, mediaType: string, thumbnailBase64: string) => void;
}
let { oncapture }: Props = $props();
let videoElement: HTMLVideoElement | null = $state(null);
let stream: MediaStream | null = $state(null);
let ready = $state(false);
async function findUltraWideCamera(): Promise<string | null> {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const videoDevices = devices.filter((d) => d.kind === 'videoinput');
const ultraWide = videoDevices.find(
(d) => d.label.toLowerCase().includes('ultra') && d.label.toLowerCase().includes('back')
);
return ultraWide?.deviceId ?? null;
} catch {
return null;
}
}
async function startCamera() {
if (!navigator.mediaDevices?.getUserMedia) return;
try {
stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: { ideal: 'environment' } },
audio: false
});
const ultraWideId = await findUltraWideCamera();
if (ultraWideId) {
stream.getTracks().forEach((t) => t.stop());
stream = await navigator.mediaDevices.getUserMedia({
video: {
deviceId: { exact: ultraWideId },
width: { ideal: 4032 },
height: { ideal: 3024 }
},
audio: false
});
} else {
stream.getTracks().forEach((t) => t.stop());
stream = await navigator.mediaDevices.getUserMedia({
video: {
facingMode: { ideal: 'environment' },
width: { ideal: 4032 },
height: { ideal: 3024 }
},
audio: false
});
}
if (videoElement) {
videoElement.srcObject = stream;
await new Promise<void>((resolve) => {
if (videoElement) {
videoElement.onloadedmetadata = () => resolve();
}
});
ready = true;
}
} catch {
ready = false;
}
}
export function capture() {
if (!ready || !videoElement) return false;
const canvas = document.createElement('canvas');
canvas.width = videoElement.videoWidth;
canvas.height = videoElement.videoHeight;
const ctx = canvas.getContext('2d');
if (!ctx) return false;
ctx.drawImage(videoElement, 0, 0);
const maxSize = 1920;
const scale = Math.min(maxSize / canvas.width, maxSize / canvas.height, 1);
const outCanvas = document.createElement('canvas');
outCanvas.width = Math.round(canvas.width * scale);
outCanvas.height = Math.round(canvas.height * scale);
const outCtx = outCanvas.getContext('2d');
if (!outCtx) return false;
outCtx.drawImage(canvas, 0, 0, outCanvas.width, outCanvas.height);
const base64 = outCanvas.toDataURL('image/jpeg', 0.65).split(',')[1];
const mediaType = 'image/jpeg';
const thumbMaxSize = 800;
const thumbScale = Math.min(thumbMaxSize / outCanvas.width, thumbMaxSize / outCanvas.height, 1);
const thumbCanvas = document.createElement('canvas');
thumbCanvas.width = Math.round(outCanvas.width * thumbScale);
thumbCanvas.height = Math.round(outCanvas.height * thumbScale);
const thumbCtx = thumbCanvas.getContext('2d');
if (thumbCtx) {
thumbCtx.drawImage(outCanvas, 0, 0, thumbCanvas.width, thumbCanvas.height);
}
const thumbnailBase64 = thumbCanvas.toDataURL('image/jpeg', 0.6).split(',')[1];
oncapture(base64, mediaType, thumbnailBase64);
return true;
}
onMount(() => {
startCamera();
return () => {
if (stream) {
stream.getTracks().forEach((track) => track.stop());
}
};
});
</script>
<div class="fixed inset-0 z-40 bg-black">
<video bind:this={videoElement} autoplay playsinline muted class="h-full w-full object-cover"
></video>
</div>

View File

@@ -20,6 +20,9 @@
function handleTouchEnd(e: TouchEvent) {
if (e.touches.length > 0) return;
const target = e.target as HTMLElement;
if (target?.closest('[data-camera-ui]')) return;
const touch = e.changedTouches[0];
const now = Date.now();
const x = touch.clientX;

View File

@@ -0,0 +1,29 @@
<script lang="ts">
import { onMount } from 'svelte';
interface Props {
oncomplete: () => void;
oncancel: () => void;
}
let { oncomplete, oncancel }: Props = $props();
let count = $state(3);
onMount(() => {
const interval = setInterval(() => {
count--;
if (count === 0) {
clearInterval(interval);
oncomplete();
}
}, 1000);
return () => clearInterval(interval);
});
</script>
<div class="fixed inset-0 z-50 flex flex-col items-center justify-center bg-black" data-camera-ui>
<span class="text-8xl font-bold text-white">{count}</span>
<button onclick={oncancel} class="mt-8 text-sm text-neutral-400">Cancel</button>
</div>

View File

@@ -3,7 +3,7 @@ import { getContext, setContext } from 'svelte';
import type { FunctionReference, FunctionArgs, FunctionReturnType } from 'convex/server';
const POLLING_CONTEXT_KEY = 'convex-polling';
const POLL_INTERVAL = 1000;
const POLL_INTERVAL = 500;
type PollingContext = {
client: ConvexHttpClient;

View File

@@ -8,18 +8,30 @@
* @module
*/
import type * as chats from '../chats.js';
import type * as messages from '../messages.js';
import type * as pendingGenerations from '../pendingGenerations.js';
import type * as users from '../users.js';
import type * as chats from "../chats.js";
import type * as devicePairings from "../devicePairings.js";
import type * as messages from "../messages.js";
import type * as pairingRequests from "../pairingRequests.js";
import type * as pendingGenerations from "../pendingGenerations.js";
import type * as photoDrafts from "../photoDrafts.js";
import type * as photoRequests from "../photoRequests.js";
import type * as users from "../users.js";
import type { ApiFromModules, FilterApi, FunctionReference } from 'convex/server';
import type {
ApiFromModules,
FilterApi,
FunctionReference,
} from "convex/server";
declare const fullApi: ApiFromModules<{
chats: typeof chats;
messages: typeof messages;
pendingGenerations: typeof pendingGenerations;
users: typeof users;
chats: typeof chats;
devicePairings: typeof devicePairings;
messages: typeof messages;
pairingRequests: typeof pairingRequests;
pendingGenerations: typeof pendingGenerations;
photoDrafts: typeof photoDrafts;
photoRequests: typeof photoRequests;
users: typeof users;
}>;
/**
@@ -30,7 +42,10 @@ declare const fullApi: ApiFromModules<{
* const myFunctionReference = api.myModule.myFunction;
* ```
*/
export declare const api: FilterApi<typeof fullApi, FunctionReference<any, 'public'>>;
export declare const api: FilterApi<
typeof fullApi,
FunctionReference<any, "public">
>;
/**
* A utility for referencing Convex functions in your app's internal API.
@@ -40,6 +55,9 @@ export declare const api: FilterApi<typeof fullApi, FunctionReference<any, 'publ
* const myFunctionReference = internal.myModule.myFunction;
* ```
*/
export declare const internal: FilterApi<typeof fullApi, FunctionReference<any, 'internal'>>;
export declare const internal: FilterApi<
typeof fullApi,
FunctionReference<any, "internal">
>;
export declare const components: {};

View File

@@ -8,7 +8,7 @@
* @module
*/
import { anyApi, componentsGeneric } from 'convex/server';
import { anyApi, componentsGeneric } from "convex/server";
/**
* A utility for referencing Convex functions in your app's API.

View File

@@ -9,13 +9,13 @@
*/
import type {
DataModelFromSchemaDefinition,
DocumentByName,
TableNamesInDataModel,
SystemTableNames
} from 'convex/server';
import type { GenericId } from 'convex/values';
import schema from '../schema.js';
DataModelFromSchemaDefinition,
DocumentByName,
TableNamesInDataModel,
SystemTableNames,
} from "convex/server";
import type { GenericId } from "convex/values";
import schema from "../schema.js";
/**
* The names of all of your Convex tables.
@@ -27,7 +27,10 @@ export type TableNames = TableNamesInDataModel<DataModel>;
*
* @typeParam TableName - A string literal type of the table name (like "users").
*/
export type Doc<TableName extends TableNames> = DocumentByName<DataModel, TableName>;
export type Doc<TableName extends TableNames> = DocumentByName<
DataModel,
TableName
>;
/**
* An identifier for a document in Convex.
@@ -42,7 +45,8 @@ export type Doc<TableName extends TableNames> = DocumentByName<DataModel, TableN
*
* @typeParam TableName - A string literal type of the table name (like "users").
*/
export type Id<TableName extends TableNames | SystemTableNames> = GenericId<TableName>;
export type Id<TableName extends TableNames | SystemTableNames> =
GenericId<TableName>;
/**
* A type describing your Convex data model.

View File

@@ -9,17 +9,17 @@
*/
import {
ActionBuilder,
HttpActionBuilder,
MutationBuilder,
QueryBuilder,
GenericActionCtx,
GenericMutationCtx,
GenericQueryCtx,
GenericDatabaseReader,
GenericDatabaseWriter
} from 'convex/server';
import type { DataModel } from './dataModel.js';
ActionBuilder,
HttpActionBuilder,
MutationBuilder,
QueryBuilder,
GenericActionCtx,
GenericMutationCtx,
GenericQueryCtx,
GenericDatabaseReader,
GenericDatabaseWriter,
} from "convex/server";
import type { DataModel } from "./dataModel.js";
/**
* Define a query in this Convex app's public API.
@@ -29,7 +29,7 @@ import type { DataModel } from './dataModel.js';
* @param func - The query function. It receives a {@link QueryCtx} as its first argument.
* @returns The wrapped query. Include this as an `export` to name it and make it accessible.
*/
export declare const query: QueryBuilder<DataModel, 'public'>;
export declare const query: QueryBuilder<DataModel, "public">;
/**
* Define a query that is only accessible from other Convex functions (but not from the client).
@@ -39,7 +39,7 @@ export declare const query: QueryBuilder<DataModel, 'public'>;
* @param func - The query function. It receives a {@link QueryCtx} as its first argument.
* @returns The wrapped query. Include this as an `export` to name it and make it accessible.
*/
export declare const internalQuery: QueryBuilder<DataModel, 'internal'>;
export declare const internalQuery: QueryBuilder<DataModel, "internal">;
/**
* Define a mutation in this Convex app's public API.
@@ -49,7 +49,7 @@ export declare const internalQuery: QueryBuilder<DataModel, 'internal'>;
* @param func - The mutation function. It receives a {@link MutationCtx} as its first argument.
* @returns The wrapped mutation. Include this as an `export` to name it and make it accessible.
*/
export declare const mutation: MutationBuilder<DataModel, 'public'>;
export declare const mutation: MutationBuilder<DataModel, "public">;
/**
* Define a mutation that is only accessible from other Convex functions (but not from the client).
@@ -59,7 +59,7 @@ export declare const mutation: MutationBuilder<DataModel, 'public'>;
* @param func - The mutation function. It receives a {@link MutationCtx} as its first argument.
* @returns The wrapped mutation. Include this as an `export` to name it and make it accessible.
*/
export declare const internalMutation: MutationBuilder<DataModel, 'internal'>;
export declare const internalMutation: MutationBuilder<DataModel, "internal">;
/**
* Define an action in this Convex app's public API.
@@ -72,7 +72,7 @@ export declare const internalMutation: MutationBuilder<DataModel, 'internal'>;
* @param func - The action. It receives an {@link ActionCtx} as its first argument.
* @returns The wrapped action. Include this as an `export` to name it and make it accessible.
*/
export declare const action: ActionBuilder<DataModel, 'public'>;
export declare const action: ActionBuilder<DataModel, "public">;
/**
* Define an action that is only accessible from other Convex functions (but not from the client).
@@ -80,7 +80,7 @@ export declare const action: ActionBuilder<DataModel, 'public'>;
* @param func - The function. It receives an {@link ActionCtx} as its first argument.
* @returns The wrapped function. Include this as an `export` to name it and make it accessible.
*/
export declare const internalAction: ActionBuilder<DataModel, 'internal'>;
export declare const internalAction: ActionBuilder<DataModel, "internal">;
/**
* Define an HTTP action.

View File

@@ -9,14 +9,14 @@
*/
import {
actionGeneric,
httpActionGeneric,
queryGeneric,
mutationGeneric,
internalActionGeneric,
internalMutationGeneric,
internalQueryGeneric
} from 'convex/server';
actionGeneric,
httpActionGeneric,
queryGeneric,
mutationGeneric,
internalActionGeneric,
internalMutationGeneric,
internalQueryGeneric,
} from "convex/server";
/**
* Define a query in this Convex app's public API.

View File

@@ -43,8 +43,15 @@ export const clear = mutation({
.collect();
for (const message of messages) {
if (args.preserveImages && message.imageBase64) {
continue;
if (args.preserveImages) {
const hasLegacyImage = message.imageBase64 || message.imagesBase64?.length;
const messageImages = await ctx.db
.query('messageImages')
.withIndex('by_message_id', (q) => q.eq('messageId', message._id))
.first();
if (hasLegacyImage || messageImages) {
continue;
}
}
await ctx.db.delete(message._id);
}

View File

@@ -0,0 +1,109 @@
import { v } from 'convex/values';
import { mutation, query } from './_generated/server';
export const register = mutation({
args: {
chatId: v.id('chats'),
deviceId: v.string(),
hasCamera: v.boolean()
},
returns: v.id('devicePairings'),
handler: async (ctx, args) => {
const existing = await ctx.db
.query('devicePairings')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.collect();
const device = existing.find((d) => d.deviceId === args.deviceId);
if (device) {
await ctx.db.patch(device._id, {
hasCamera: args.hasCamera,
lastSeen: Date.now()
});
return device._id;
}
return await ctx.db.insert('devicePairings', {
chatId: args.chatId,
deviceId: args.deviceId,
hasCamera: args.hasCamera,
lastSeen: Date.now()
});
}
});
export const heartbeat = mutation({
args: {
chatId: v.id('chats'),
deviceId: v.string()
},
returns: v.null(),
handler: async (ctx, args) => {
const devices = await ctx.db
.query('devicePairings')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.collect();
const device = devices.find((d) => d.deviceId === args.deviceId);
if (device) {
await ctx.db.patch(device._id, { lastSeen: Date.now() });
}
return null;
}
});
export const getMyDevice = query({
args: { chatId: v.id('chats'), deviceId: v.string() },
returns: v.union(
v.object({
_id: v.id('devicePairings'),
_creationTime: v.number(),
chatId: v.id('chats'),
deviceId: v.string(),
hasCamera: v.boolean(),
pairedWithDeviceId: v.optional(v.string()),
lastSeen: v.number()
}),
v.null()
),
handler: async (ctx, args) => {
const devices = await ctx.db
.query('devicePairings')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.collect();
return devices.find((d) => d.deviceId === args.deviceId) ?? null;
}
});
export const getPairedDevice = query({
args: { chatId: v.id('chats'), deviceId: v.string() },
returns: v.union(
v.object({
_id: v.id('devicePairings'),
_creationTime: v.number(),
chatId: v.id('chats'),
deviceId: v.string(),
hasCamera: v.boolean(),
pairedWithDeviceId: v.optional(v.string()),
lastSeen: v.number()
}),
v.null()
),
handler: async (ctx, args) => {
const devices = await ctx.db
.query('devicePairings')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.collect();
const myDevice = devices.find((d) => d.deviceId === args.deviceId);
if (!myDevice?.pairedWithDeviceId) return null;
const thirtySecondsAgo = Date.now() - 30000;
const paired = devices.find(
(d) => d.deviceId === myDevice.pairedWithDeviceId && d.lastSeen > thirtySecondsAgo
);
return paired ?? null;
}
});

View File

@@ -1,5 +1,6 @@
import { v } from 'convex/values';
import { mutation, query } from './_generated/server';
import type { Id } from './_generated/dataModel';
export const listByChat = query({
args: { chatId: v.id('chats') },
@@ -39,6 +40,7 @@ export const create = mutation({
imageMediaType: v.optional(v.string()),
imagesBase64: v.optional(v.array(v.string())),
imagesMediaTypes: v.optional(v.array(v.string())),
photoDraftIds: v.optional(v.array(v.id('photoDrafts'))),
followUpOptions: v.optional(v.array(v.string())),
isStreaming: v.optional(v.boolean())
},
@@ -58,18 +60,50 @@ export const create = mutation({
isStreaming: args.isStreaming
});
const drafts: Array<{ base64: string; mediaType: string; id: Id<'photoDrafts'> }> = [];
if (args.photoDraftIds && args.photoDraftIds.length > 0) {
for (const draftId of args.photoDraftIds) {
const draft = await ctx.db.get(draftId);
if (draft) {
drafts.push({ base64: draft.base64, mediaType: draft.mediaType, id: draft._id });
}
}
}
for (let i = 0; i < drafts.length; i++) {
await ctx.db.insert('messageImages', {
messageId,
base64: drafts[i].base64,
mediaType: drafts[i].mediaType,
order: i
});
}
if (args.source === 'web' && args.role === 'user') {
const chat = await ctx.db.get(args.chatId);
if (chat) {
await ctx.db.insert('pendingGenerations', {
const pendingGenId = await ctx.db.insert('pendingGenerations', {
userId: chat.userId,
chatId: args.chatId,
userMessage: args.content,
createdAt: Date.now()
});
for (let i = 0; i < drafts.length; i++) {
await ctx.db.insert('pendingGenerationImages', {
pendingGenerationId: pendingGenId,
base64: drafts[i].base64,
mediaType: drafts[i].mediaType,
order: i
});
}
}
}
for (const draft of drafts) {
await ctx.db.delete(draft.id);
}
return messageId;
}
});
@@ -177,6 +211,15 @@ export const getChatImages = query({
const images: Array<{ base64: string; mediaType: string }> = [];
for (const m of messages) {
const msgImages = await ctx.db
.query('messageImages')
.withIndex('by_message_id', (q) => q.eq('messageId', m._id))
.collect();
for (const img of msgImages.sort((a, b) => a.order - b.order)) {
images.push({ base64: img.base64, mediaType: img.mediaType });
}
if (m.imagesBase64 && m.imagesMediaTypes) {
for (let i = 0; i < m.imagesBase64.length; i++) {
images.push({

View File

@@ -0,0 +1,122 @@
import { v } from 'convex/values';
import { mutation, query } from './_generated/server';
export const create = mutation({
args: {
chatId: v.id('chats'),
fromDeviceId: v.string()
},
returns: v.id('pairingRequests'),
handler: async (ctx, args) => {
const existing = await ctx.db
.query('pairingRequests')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.collect();
const pending = existing.find(
(r) => r.fromDeviceId === args.fromDeviceId && r.status === 'pending'
);
if (pending) return pending._id;
return await ctx.db.insert('pairingRequests', {
chatId: args.chatId,
fromDeviceId: args.fromDeviceId,
status: 'pending',
createdAt: Date.now()
});
}
});
export const accept = mutation({
args: {
requestId: v.id('pairingRequests'),
acceptingDeviceId: v.string()
},
returns: v.null(),
handler: async (ctx, args) => {
const request = await ctx.db.get(args.requestId);
if (!request || request.status !== 'pending') return null;
await ctx.db.patch(args.requestId, { status: 'accepted' });
const devices = await ctx.db
.query('devicePairings')
.withIndex('by_chat_id', (q) => q.eq('chatId', request.chatId))
.collect();
const fromDevice = devices.find((d) => d.deviceId === request.fromDeviceId);
const acceptingDevice = devices.find((d) => d.deviceId === args.acceptingDeviceId);
if (fromDevice) {
await ctx.db.patch(fromDevice._id, { pairedWithDeviceId: args.acceptingDeviceId });
}
if (acceptingDevice) {
await ctx.db.patch(acceptingDevice._id, { pairedWithDeviceId: request.fromDeviceId });
}
return null;
}
});
export const reject = mutation({
args: { requestId: v.id('pairingRequests') },
returns: v.null(),
handler: async (ctx, args) => {
const request = await ctx.db.get(args.requestId);
if (!request || request.status !== 'pending') return null;
await ctx.db.patch(args.requestId, { status: 'rejected' });
return null;
}
});
export const getPending = query({
args: { chatId: v.id('chats'), excludeDeviceId: v.string() },
returns: v.union(
v.object({
_id: v.id('pairingRequests'),
_creationTime: v.number(),
chatId: v.id('chats'),
fromDeviceId: v.string(),
status: v.union(v.literal('pending'), v.literal('accepted'), v.literal('rejected')),
createdAt: v.number()
}),
v.null()
),
handler: async (ctx, args) => {
const requests = await ctx.db
.query('pairingRequests')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.collect();
return (
requests.find((r) => r.status === 'pending' && r.fromDeviceId !== args.excludeDeviceId) ??
null
);
}
});
export const unpair = mutation({
args: {
chatId: v.id('chats'),
deviceId: v.string()
},
returns: v.null(),
handler: async (ctx, args) => {
const devices = await ctx.db
.query('devicePairings')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.collect();
const myDevice = devices.find((d) => d.deviceId === args.deviceId);
if (!myDevice?.pairedWithDeviceId) return null;
const pairedDevice = devices.find((d) => d.deviceId === myDevice.pairedWithDeviceId);
await ctx.db.patch(myDevice._id, { pairedWithDeviceId: undefined });
if (pairedDevice) {
await ctx.db.patch(pairedDevice._id, { pairedWithDeviceId: undefined });
}
return null;
}
});

View File

@@ -10,11 +10,33 @@ export const list = query({
userId: v.id('users'),
chatId: v.id('chats'),
userMessage: v.string(),
imagesBase64: v.optional(v.array(v.string())),
imagesMediaTypes: v.optional(v.array(v.string())),
createdAt: v.number()
})
),
handler: async (ctx) => {
return await ctx.db.query('pendingGenerations').collect();
const pending = await ctx.db.query('pendingGenerations').collect();
const result = [];
for (const p of pending) {
const images = await ctx.db
.query('pendingGenerationImages')
.withIndex('by_pending_generation_id', (q) => q.eq('pendingGenerationId', p._id))
.collect();
const sortedImages = images.sort((a, b) => a.order - b.order);
result.push({
...p,
imagesBase64:
sortedImages.length > 0 ? sortedImages.map((img) => img.base64) : p.imagesBase64,
imagesMediaTypes:
sortedImages.length > 0 ? sortedImages.map((img) => img.mediaType) : p.imagesMediaTypes
});
}
return result;
}
});
@@ -39,7 +61,35 @@ export const remove = mutation({
args: { id: v.id('pendingGenerations') },
returns: v.null(),
handler: async (ctx, args) => {
const images = await ctx.db
.query('pendingGenerationImages')
.withIndex('by_pending_generation_id', (q) => q.eq('pendingGenerationId', args.id))
.collect();
for (const img of images) {
await ctx.db.delete(img._id);
}
await ctx.db.delete(args.id);
return null;
}
});
export const getImages = query({
args: { pendingGenerationId: v.id('pendingGenerations') },
returns: v.array(
v.object({
base64: v.string(),
mediaType: v.string()
})
),
handler: async (ctx, args) => {
const images = await ctx.db
.query('pendingGenerationImages')
.withIndex('by_pending_generation_id', (q) =>
q.eq('pendingGenerationId', args.pendingGenerationId)
)
.collect();
return images
.sort((a, b) => a.order - b.order)
.map((img) => ({ base64: img.base64, mediaType: img.mediaType }));
}
});

View File

@@ -0,0 +1,95 @@
import { v } from 'convex/values';
import { mutation, query } from './_generated/server';
const photoValidator = v.object({
base64: v.string(),
mediaType: v.string()
});
export const get = query({
args: { chatId: v.id('chats'), deviceId: v.string() },
returns: v.object({
photos: v.array(
v.object({
_id: v.id('photoDrafts'),
base64: v.string(),
mediaType: v.string()
})
)
}),
handler: async (ctx, args) => {
const drafts = await ctx.db
.query('photoDrafts')
.withIndex('by_chat_id_and_device_id', (q) =>
q.eq('chatId', args.chatId).eq('deviceId', args.deviceId)
)
.collect();
return {
photos: drafts.map((d) => ({
_id: d._id,
base64: d.base64,
mediaType: d.mediaType
}))
};
}
});
export const addPhoto = mutation({
args: {
chatId: v.id('chats'),
deviceId: v.string(),
photo: photoValidator
},
returns: v.null(),
handler: async (ctx, args) => {
await ctx.db.insert('photoDrafts', {
chatId: args.chatId,
deviceId: args.deviceId,
base64: args.photo.base64,
mediaType: args.photo.mediaType,
createdAt: Date.now()
});
return null;
}
});
export const removePhoto = mutation({
args: {
chatId: v.id('chats'),
deviceId: v.string(),
index: v.number()
},
returns: v.null(),
handler: async (ctx, args) => {
const drafts = await ctx.db
.query('photoDrafts')
.withIndex('by_chat_id_and_device_id', (q) =>
q.eq('chatId', args.chatId).eq('deviceId', args.deviceId)
)
.collect();
if (drafts[args.index]) {
await ctx.db.delete(drafts[args.index]._id);
}
return null;
}
});
export const clear = mutation({
args: { chatId: v.id('chats'), deviceId: v.string() },
returns: v.null(),
handler: async (ctx, args) => {
const drafts = await ctx.db
.query('photoDrafts')
.withIndex('by_chat_id_and_device_id', (q) =>
q.eq('chatId', args.chatId).eq('deviceId', args.deviceId)
)
.collect();
for (const draft of drafts) {
await ctx.db.delete(draft._id);
}
return null;
}
});

View File

@@ -0,0 +1,171 @@
import { v } from 'convex/values';
import { mutation, query } from './_generated/server';
const photoRequestValidator = v.object({
_id: v.id('photoRequests'),
_creationTime: v.number(),
chatId: v.id('chats'),
requesterId: v.string(),
captureDeviceId: v.optional(v.string()),
status: v.union(
v.literal('pending'),
v.literal('countdown'),
v.literal('capture_now'),
v.literal('captured'),
v.literal('accepted'),
v.literal('rejected')
),
photoBase64: v.optional(v.string()),
photoMediaType: v.optional(v.string()),
thumbnailBase64: v.optional(v.string()),
createdAt: v.number()
});
export const create = mutation({
args: {
chatId: v.id('chats'),
requesterId: v.string(),
captureDeviceId: v.string()
},
returns: v.id('photoRequests'),
handler: async (ctx, args) => {
return await ctx.db.insert('photoRequests', {
chatId: args.chatId,
requesterId: args.requesterId,
captureDeviceId: args.captureDeviceId,
status: 'countdown',
createdAt: Date.now()
});
}
});
export const markCaptureNow = mutation({
args: { requestId: v.id('photoRequests') },
returns: v.null(),
handler: async (ctx, args) => {
await ctx.db.patch(args.requestId, { status: 'capture_now' });
return null;
}
});
export const submitPhoto = mutation({
args: {
requestId: v.id('photoRequests'),
photoBase64: v.string(),
photoMediaType: v.string(),
thumbnailBase64: v.optional(v.string())
},
returns: v.null(),
handler: async (ctx, args) => {
await ctx.db.patch(args.requestId, {
status: 'captured',
photoBase64: args.photoBase64,
photoMediaType: args.photoMediaType,
thumbnailBase64: args.thumbnailBase64
});
return null;
}
});
export const markAccepted = mutation({
args: { requestId: v.id('photoRequests') },
returns: v.null(),
handler: async (ctx, args) => {
await ctx.db.patch(args.requestId, { status: 'accepted' });
return null;
}
});
export const markRejected = mutation({
args: { requestId: v.id('photoRequests') },
returns: v.null(),
handler: async (ctx, args) => {
await ctx.db.patch(args.requestId, { status: 'rejected' });
return null;
}
});
export const getCaptureNowRequest = query({
args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) },
returns: v.union(photoRequestValidator, v.null()),
handler: async (ctx, args) => {
const requests = await ctx.db
.query('photoRequests')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.order('desc')
.take(50);
return requests.find((r) => r.status === 'capture_now') ?? null;
}
});
export const getActiveForCapture = query({
args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) },
returns: v.union(photoRequestValidator, v.null()),
handler: async (ctx, args) => {
const requests = await ctx.db
.query('photoRequests')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.order('desc')
.take(50);
return requests.find((r) => r.status === 'countdown' || r.status === 'capture_now') ?? null;
}
});
export const getMyActiveRequest = query({
args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) },
returns: v.union(photoRequestValidator, v.null()),
handler: async (ctx, args) => {
const requests = await ctx.db
.query('photoRequests')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.take(100);
if (!args.deviceId) return null;
return (
requests.find(
(r) =>
r.requesterId === args.deviceId &&
(r.status === 'countdown' || r.status === 'capture_now' || r.status === 'captured')
) ?? null
);
}
});
export const cleanup = mutation({
args: { chatId: v.id('chats') },
returns: v.number(),
handler: async (ctx, args) => {
const requests = await ctx.db
.query('photoRequests')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.take(20);
let deleted = 0;
for (const req of requests) {
await ctx.db.delete(req._id);
deleted++;
}
return deleted;
}
});
export const getCapturedForPhone = query({
args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) },
returns: v.union(photoRequestValidator, v.null()),
handler: async (ctx, args) => {
const requests = await ctx.db
.query('photoRequests')
.withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId))
.order('desc')
.take(50);
if (!args.deviceId) return null;
return (
requests.find((r) => r.captureDeviceId === args.deviceId && r.status === 'captured') ?? null
);
}
});

View File

@@ -40,6 +40,63 @@ export default defineSchema({
userId: v.id('users'),
chatId: v.id('chats'),
userMessage: v.string(),
imagesBase64: v.optional(v.array(v.string())),
imagesMediaTypes: v.optional(v.array(v.string())),
createdAt: v.number()
})
}),
pendingGenerationImages: defineTable({
pendingGenerationId: v.id('pendingGenerations'),
base64: v.string(),
mediaType: v.string(),
order: v.number()
}).index('by_pending_generation_id', ['pendingGenerationId']),
messageImages: defineTable({
messageId: v.id('messages'),
base64: v.string(),
mediaType: v.string(),
order: v.number()
}).index('by_message_id', ['messageId']),
devicePairings: defineTable({
chatId: v.id('chats'),
deviceId: v.string(),
hasCamera: v.boolean(),
pairedWithDeviceId: v.optional(v.string()),
lastSeen: v.number()
}).index('by_chat_id', ['chatId']),
pairingRequests: defineTable({
chatId: v.id('chats'),
fromDeviceId: v.string(),
status: v.union(v.literal('pending'), v.literal('accepted'), v.literal('rejected')),
createdAt: v.number()
}).index('by_chat_id', ['chatId']),
photoRequests: defineTable({
chatId: v.id('chats'),
requesterId: v.string(),
captureDeviceId: v.optional(v.string()),
status: v.union(
v.literal('pending'),
v.literal('countdown'),
v.literal('capture_now'),
v.literal('captured'),
v.literal('accepted'),
v.literal('rejected')
),
photoBase64: v.optional(v.string()),
photoMediaType: v.optional(v.string()),
thumbnailBase64: v.optional(v.string()),
createdAt: v.number()
}).index('by_chat_id', ['chatId']),
photoDrafts: defineTable({
chatId: v.id('chats'),
deviceId: v.string(),
base64: v.string(),
mediaType: v.string(),
createdAt: v.number()
}).index('by_chat_id_and_device_id', ['chatId', 'deviceId'])
});

View File

@@ -1,13 +1,21 @@
<script lang="ts">
import { page } from '$app/state';
import { getContext } from 'svelte';
import { browser } from '$app/environment';
import { getContext, onMount } from 'svelte';
import { SvelteSet } from 'svelte/reactivity';
import { useQuery, useConvexClient } from 'convex-svelte';
import { usePollingQuery, usePollingMutation } from '$lib/convex-polling.svelte';
import { api } from '$lib/convex/_generated/api';
import type { Id } from '$lib/convex/_generated/dataModel';
import ChatMessage from '$lib/components/ChatMessage.svelte';
import ChatInput from '$lib/components/ChatInput.svelte';
import FollowUpButtons from '$lib/components/FollowUpButtons.svelte';
import StealthOverlay from '$lib/components/StealthOverlay.svelte';
import CameraCapture from '$lib/components/CameraCapture.svelte';
import WatchCountdown from '$lib/components/WatchCountdown.svelte';
import PhotoPreview from '$lib/components/PhotoPreview.svelte';
import DraftBadge from '$lib/components/DraftBadge.svelte';
import SilentCapture from '$lib/components/SilentCapture.svelte';
const usePolling = getContext<boolean>('convex-use-polling') ?? false;
let mnemonic = $derived(page.params.mnemonic);
@@ -15,6 +23,60 @@
let lastMessageElement: HTMLDivElement | null = $state(null);
let showScrollButton = $state(false);
let deviceId = $state('');
let hasCamera = $state(false);
let showCamera = $state(false);
let showWatchCountdown = $state(false);
let activeRequestId: Id<'photoRequests'> | null = $state(null);
let previewPhoto: {
thumbnail: string;
fullBase64: string;
mediaType: string;
requestId: Id<'photoRequests'>;
} | null = $state(null);
let shownPreviewIds = new SvelteSet<string>();
let silentCaptureRef: SilentCapture | null = $state(null);
let processedCaptureNowIds = new SvelteSet<string>();
function generateId(): string {
if (typeof crypto !== 'undefined' && crypto.randomUUID) {
return crypto.randomUUID();
}
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
function getOrCreateDeviceId(): string {
if (!browser) return '';
let id = localStorage.getItem('stealth-device-id');
if (!id) {
id = generateId();
localStorage.setItem('stealth-device-id', id);
}
return id;
}
async function checkCamera(): Promise<boolean> {
if (!browser) return false;
if (!navigator.mediaDevices?.enumerateDevices) return false;
try {
const devices = await navigator.mediaDevices.enumerateDevices();
return devices.some((d) => d.kind === 'videoinput');
} catch {
return false;
}
}
onMount(() => {
deviceId = getOrCreateDeviceId();
checkCamera().then((has) => {
hasCamera = has;
});
});
$effect(() => {
if (!lastMessageElement) return;
@@ -41,15 +103,13 @@
: null;
const chatData = $derived(usePolling ? chatDataPoll! : chatDataWs!);
const chatId = $derived(chatData.data?.chat?._id);
const messagesQueryWs = usePolling
? null
: useQuery(api.messages.listByChat, () =>
chatData.data?.chat?._id ? { chatId: chatData.data.chat._id } : 'skip'
);
: useQuery(api.messages.listByChat, () => (chatId ? { chatId } : 'skip'));
const messagesQueryPoll = usePolling
? usePollingQuery(api.messages.listByChat, () =>
chatData.data?.chat?._id ? { chatId: chatData.data.chat._id } : 'skip'
)
? usePollingQuery(api.messages.listByChat, () => (chatId ? { chatId } : 'skip'))
: null;
const messagesQuery = $derived(usePolling ? messagesQueryPoll! : messagesQueryWs!);
@@ -61,6 +121,103 @@
: []
);
const myDeviceWs = usePolling
? null
: useQuery(api.devicePairings.getMyDevice, () =>
chatId && deviceId ? { chatId, deviceId } : 'skip'
);
const myDevicePoll = usePolling
? usePollingQuery(api.devicePairings.getMyDevice, () =>
chatId && deviceId ? { chatId, deviceId } : 'skip'
)
: null;
const myDevice = $derived(usePolling ? myDevicePoll! : myDeviceWs!);
const pairedDeviceWs = usePolling
? null
: useQuery(api.devicePairings.getPairedDevice, () =>
chatId && deviceId ? { chatId, deviceId } : 'skip'
);
const pairedDevicePoll = usePolling
? usePollingQuery(api.devicePairings.getPairedDevice, () =>
chatId && deviceId ? { chatId, deviceId } : 'skip'
)
: null;
const pairedDevice = $derived(usePolling ? pairedDevicePoll! : pairedDeviceWs!);
const isPaired = $derived(!!myDevice.data?.pairedWithDeviceId && !!pairedDevice.data);
const pendingPairingWs = usePolling
? null
: useQuery(api.pairingRequests.getPending, () =>
chatId && deviceId ? { chatId, excludeDeviceId: deviceId } : 'skip'
);
const pendingPairingPoll = usePolling
? usePollingQuery(api.pairingRequests.getPending, () =>
chatId && deviceId ? { chatId, excludeDeviceId: deviceId } : 'skip'
)
: null;
const pendingPairing = $derived(usePolling ? pendingPairingPoll! : pendingPairingWs!);
const captureNowRequestWs = usePolling
? null
: useQuery(api.photoRequests.getCaptureNowRequest, () => (chatId ? { chatId } : 'skip'));
const captureNowRequestPoll = usePolling
? usePollingQuery(api.photoRequests.getCaptureNowRequest, () => (chatId ? { chatId } : 'skip'))
: null;
const captureNowRequest = $derived(usePolling ? captureNowRequestPoll! : captureNowRequestWs!);
const myActiveRequestWs = usePolling
? null
: useQuery(api.photoRequests.getMyActiveRequest, () =>
chatId && deviceId ? { chatId, deviceId } : 'skip'
);
const myActiveRequestPoll = usePolling
? usePollingQuery(api.photoRequests.getMyActiveRequest, () =>
chatId && deviceId ? { chatId, deviceId } : 'skip'
)
: null;
const myActiveRequest = $derived(usePolling ? myActiveRequestPoll! : myActiveRequestWs!);
const photoDraftWs = usePolling
? null
: useQuery(api.photoDrafts.get, () => (chatId && deviceId ? { chatId, deviceId } : 'skip'));
const photoDraftPoll = usePolling
? usePollingQuery(api.photoDrafts.get, () =>
chatId && deviceId ? { chatId, deviceId } : 'skip'
)
: null;
const photoDraft = $derived(usePolling ? photoDraftPoll! : photoDraftWs!);
const draftPhotos = $derived(photoDraft.data?.photos ?? []);
$effect(() => {
const req = captureNowRequest.data;
if (req && hasCamera && !processedCaptureNowIds.has(req._id)) {
processedCaptureNowIds.add(req._id);
const tryCapture = () => {
const success = silentCaptureRef?.capture();
if (!success) {
setTimeout(tryCapture, 100);
}
};
tryCapture();
}
});
$effect(() => {
const req = myActiveRequest.data;
if (req?.status === 'captured' && req.photoBase64 && req.photoMediaType) {
if (shownPreviewIds.has(req._id)) return;
shownPreviewIds.add(req._id);
previewPhoto = {
thumbnail: req.thumbnailBase64 || req.photoBase64,
fullBase64: req.photoBase64,
mediaType: req.photoMediaType,
requestId: req._id
};
}
});
let prevMessageCount = 0;
let prevLastMessageId: string | undefined;
@@ -76,45 +233,283 @@
const clientWs = usePolling ? null : useConvexClient();
const createMessagePoll = usePolling ? usePollingMutation(api.messages.create) : null;
const registerDevicePoll = usePolling ? usePollingMutation(api.devicePairings.register) : null;
const heartbeatPoll = usePolling ? usePollingMutation(api.devicePairings.heartbeat) : null;
const addPhotoPoll = usePolling ? usePollingMutation(api.photoDrafts.addPhoto) : null;
const removePhotoPoll = usePolling ? usePollingMutation(api.photoDrafts.removePhoto) : null;
const createPairingPoll = usePolling ? usePollingMutation(api.pairingRequests.create) : null;
const acceptPairingPoll = usePolling ? usePollingMutation(api.pairingRequests.accept) : null;
const rejectPairingPoll = usePolling ? usePollingMutation(api.pairingRequests.reject) : null;
const unpairPoll = usePolling ? usePollingMutation(api.pairingRequests.unpair) : null;
const createRequestPoll = usePolling ? usePollingMutation(api.photoRequests.create) : null;
const markCaptureNowPoll = usePolling
? usePollingMutation(api.photoRequests.markCaptureNow)
: null;
const submitPhotoPoll = usePolling ? usePollingMutation(api.photoRequests.submitPhoto) : null;
const markAcceptedPoll = usePolling ? usePollingMutation(api.photoRequests.markAccepted) : null;
const markRejectedPoll = usePolling ? usePollingMutation(api.photoRequests.markRejected) : null;
async function sendMessage(content: string) {
$effect(() => {
if (!chatId || !deviceId) return;
if (usePolling && registerDevicePoll) {
registerDevicePoll({ chatId, deviceId, hasCamera });
} else if (clientWs) {
clientWs.mutation(api.devicePairings.register, {
chatId,
deviceId,
hasCamera
});
}
const interval = setInterval(() => {
if (usePolling && heartbeatPoll) {
heartbeatPoll({ chatId, deviceId });
} else if (clientWs) {
clientWs.mutation(api.devicePairings.heartbeat, { chatId, deviceId });
}
}, 10000);
return () => clearInterval(interval);
});
function sendMessage(content: string) {
const chat = chatData.data?.chat;
if (!chat) return;
const photos = draftPhotos;
const photoDraftIds = photos.length > 0 ? photos.map((p) => p._id) : undefined;
const messageContent =
content || (photos.length > 0 ? 'Process images according to your task' : '');
if (!messageContent) return;
if (usePolling && createMessagePoll) {
createMessagePoll({
chatId: chat._id,
role: 'user',
content: messageContent,
source: 'web',
photoDraftIds
});
} else if (clientWs) {
clientWs.mutation(api.messages.create, {
chatId: chat._id,
role: 'user',
content: messageContent,
source: 'web',
photoDraftIds
});
}
}
function summarize() {
const chat = chatData.data?.chat;
if (!chat) return;
if (usePolling && createMessagePoll) {
await createMessagePoll({
createMessagePoll({
chatId: chat._id,
role: 'user',
content,
content: '/summarize',
source: 'web'
});
} else if (clientWs) {
await clientWs.mutation(api.messages.create, {
clientWs.mutation(api.messages.create, {
chatId: chat._id,
role: 'user',
content,
content: '/summarize',
source: 'web'
});
}
}
async function summarize() {
const chat = chatData.data?.chat;
if (!chat) return;
function handleTakePhoto() {
showCamera = true;
}
if (usePolling && createMessagePoll) {
await createMessagePoll({
chatId: chat._id,
role: 'user',
content: '/summarize',
source: 'web'
function handleCameraCapture(base64: string, mediaType: string) {
showCamera = false;
if (!chatId) return;
if (usePolling && addPhotoPoll) {
addPhotoPoll({ chatId, deviceId, photo: { base64, mediaType } });
} else if (clientWs) {
clientWs.mutation(api.photoDrafts.addPhoto, {
chatId,
deviceId,
photo: { base64, mediaType }
});
}
}
function handleCameraClose() {
showCamera = false;
}
function handlePair() {
if (!chatId) return;
if (usePolling && createPairingPoll) {
createPairingPoll({ chatId, fromDeviceId: deviceId });
} else if (clientWs) {
clientWs.mutation(api.pairingRequests.create, {
chatId,
fromDeviceId: deviceId
});
}
}
function handleAcceptPairing() {
const req = pendingPairing.data;
if (!req) return;
if (usePolling && acceptPairingPoll) {
acceptPairingPoll({ requestId: req._id, acceptingDeviceId: deviceId });
} else if (clientWs) {
clientWs.mutation(api.pairingRequests.accept, {
requestId: req._id,
acceptingDeviceId: deviceId
});
}
}
function handleRejectPairing() {
const req = pendingPairing.data;
if (!req) return;
if (usePolling && rejectPairingPoll) {
rejectPairingPoll({ requestId: req._id });
} else if (clientWs) {
clientWs.mutation(api.pairingRequests.reject, { requestId: req._id });
}
}
function handleUnpair() {
if (!chatId) return;
if (usePolling && unpairPoll) {
unpairPoll({ chatId, deviceId });
} else if (clientWs) {
clientWs.mutation(api.pairingRequests.unpair, {
chatId,
deviceId
});
}
}
function handleRequestPhoto() {
if (!chatId || !pairedDevice.data) return;
const captureDeviceId = pairedDevice.data.deviceId;
if (usePolling && createRequestPoll) {
createRequestPoll({ chatId, requesterId: deviceId, captureDeviceId }).then((id) => {
if (id) {
activeRequestId = id as Id<'photoRequests'>;
showWatchCountdown = true;
}
});
} else if (clientWs) {
await clientWs.mutation(api.messages.create, {
chatId: chat._id,
role: 'user',
content: '/summarize',
source: 'web'
clientWs
.mutation(api.photoRequests.create, {
chatId,
requesterId: deviceId,
captureDeviceId
})
.then((id) => {
activeRequestId = id;
showWatchCountdown = true;
});
}
}
function handleWatchCountdownComplete() {
showWatchCountdown = false;
if (!activeRequestId) return;
const reqId = activeRequestId;
activeRequestId = null;
if (usePolling && markCaptureNowPoll) {
markCaptureNowPoll({ requestId: reqId });
} else if (clientWs) {
clientWs.mutation(api.photoRequests.markCaptureNow, { requestId: reqId });
}
}
function handleWatchCountdownCancel() {
showWatchCountdown = false;
if (activeRequestId && markRejectedPoll) {
if (usePolling) {
markRejectedPoll({ requestId: activeRequestId });
} else if (clientWs) {
clientWs.mutation(api.photoRequests.markRejected, { requestId: activeRequestId });
}
}
activeRequestId = null;
}
function handleSilentCapture(base64: string, mediaType: string, thumbnailBase64: string) {
const req = captureNowRequest.data;
if (!req) return;
if (usePolling && submitPhotoPoll) {
submitPhotoPoll({
requestId: req._id,
photoBase64: base64,
photoMediaType: mediaType,
thumbnailBase64
});
} else if (clientWs) {
clientWs.mutation(api.photoRequests.submitPhoto, {
requestId: req._id,
photoBase64: base64,
photoMediaType: mediaType,
thumbnailBase64
});
}
}
function handlePreviewAccept() {
if (!previewPhoto || !chatId) return;
const photo = { base64: previewPhoto.fullBase64, mediaType: previewPhoto.mediaType };
const reqId = previewPhoto.requestId;
previewPhoto = null;
if (usePolling && addPhotoPoll && markAcceptedPoll) {
addPhotoPoll({ chatId, deviceId, photo });
markAcceptedPoll({ requestId: reqId });
} else if (clientWs) {
clientWs.mutation(api.photoDrafts.addPhoto, {
chatId,
deviceId,
photo
});
clientWs.mutation(api.photoRequests.markAccepted, {
requestId: reqId
});
}
}
function handlePreviewReject() {
if (!previewPhoto) return;
const reqId = previewPhoto.requestId;
previewPhoto = null;
if (usePolling && markRejectedPoll) {
markRejectedPoll({ requestId: reqId });
} else if (clientWs) {
clientWs.mutation(api.photoRequests.markRejected, {
requestId: reqId
});
}
}
function handleRemoveDraftPhoto(index: number) {
if (!chatId) return;
if (usePolling && removePhotoPoll) {
removePhotoPoll({ chatId, deviceId, index });
} else if (clientWs) {
clientWs.mutation(api.photoDrafts.removePhoto, {
chatId,
deviceId,
index
});
}
}
@@ -165,16 +560,50 @@
</div>
{/if}
<div class="mt-2 flex gap-1">
<button
onclick={summarize}
class="shrink-0 rounded bg-neutral-800 px-1.5 py-0.5 text-[8px] text-neutral-400"
>
/sum
</button>
<div class="flex-1">
<ChatInput onsubmit={sendMessage} />
<div class="mt-3 space-y-2">
<div class="flex gap-2">
{#if hasCamera}
<button
onclick={handleTakePhoto}
class="flex-1 rounded bg-neutral-800 py-2 text-xs text-neutral-300"
>
+ photo
</button>
{/if}
{#if isPaired && pairedDevice.data?.hasCamera}
<button
onclick={handleRequestPhoto}
class="flex-1 rounded bg-neutral-800 py-2 text-xs text-neutral-300"
>
request
</button>
{/if}
{#if isPaired}
<button
onclick={handleUnpair}
class="flex-1 rounded bg-red-900/50 py-2 text-xs text-red-300"
>
unpair
</button>
{:else}
<button
onclick={handlePair}
class="flex-1 rounded bg-neutral-800 py-2 text-xs text-neutral-300"
>
pair
</button>
{/if}
<button
onclick={summarize}
class="flex-1 rounded bg-neutral-800 py-2 text-xs text-neutral-300"
>
/sum
</button>
</div>
{#if draftPhotos.length > 0}
<DraftBadge photos={draftPhotos} onremove={handleRemoveDraftPhoto} />
{/if}
<ChatInput onsubmit={sendMessage} allowEmpty={draftPhotos.length > 0} />
</div>
{/if}
@@ -188,4 +617,50 @@
{/if}
<StealthOverlay />
{#if showCamera}
<CameraCapture oncapture={handleCameraCapture} onclose={handleCameraClose} />
{/if}
{#if pendingPairing.data && !isPaired}
<div class="fixed inset-0 z-50 flex items-center justify-center bg-black/90" data-camera-ui>
<div class="rounded-lg bg-neutral-900 p-6 text-center">
<p class="mb-4 text-sm text-white">Accept pairing request?</p>
<div class="flex gap-3">
<button
onclick={handleAcceptPairing}
class="flex-1 rounded bg-blue-600 py-2 text-sm text-white"
>
Accept
</button>
<button
onclick={handleRejectPairing}
class="flex-1 rounded bg-neutral-700 py-2 text-sm text-white"
>
Reject
</button>
</div>
</div>
</div>
{/if}
{#if showWatchCountdown}
<WatchCountdown
oncomplete={handleWatchCountdownComplete}
oncancel={handleWatchCountdownCancel}
/>
{/if}
{#if previewPhoto}
<PhotoPreview
base64={previewPhoto.thumbnail}
mediaType={previewPhoto.mediaType}
onaccept={handlePreviewAccept}
onreject={handlePreviewReject}
/>
{/if}
{#if hasCamera && isPaired}
<SilentCapture bind:this={silentCaptureRef} oncapture={handleSilentCapture} />
{/if}
</div>

View File

@@ -2,4 +2,7 @@ import tailwindcss from '@tailwindcss/vite';
import { sveltekit } from '@sveltejs/kit/vite';
import { defineConfig } from 'vite';
export default defineConfig({ plugins: [tailwindcss(), sveltekit()] });
export default defineConfig({
plugins: [tailwindcss(), sveltekit()],
server: { allowedHosts: ['porter-davidson-fibre-handhelds.trycloudflare.com'] }
});