diff --git a/backend/src/bot/handlers/message/handler.py b/backend/src/bot/handlers/message/handler.py index 68d7634..9e73043 100644 --- a/backend/src/bot/handlers/message/handler.py +++ b/backend/src/bot/handlers/message/handler.py @@ -9,6 +9,8 @@ from typing import Any from aiogram import BaseMiddleware, Bot, F, Router, html, types from aiogram.enums import ChatAction from aiogram.types import ( + BufferedInputFile, + InputMediaPhoto, KeyboardButton, ReplyKeyboardMarkup, ReplyKeyboardRemove, @@ -169,8 +171,13 @@ async def send_long_message( ) -async def process_message_from_web( # noqa: C901, PLR0912, PLR0915 - convex_user_id: str, text: str, bot: Bot, convex_chat_id: str +async def process_message_from_web( # noqa: C901, PLR0912, PLR0913, PLR0915 + convex_user_id: str, + text: str, + bot: Bot, + convex_chat_id: str, + images_base64: list[str] | None = None, + images_media_types: list[str] | None = None, ) -> None: user = await convex.query("users:getById", {"userId": convex_user_id}) @@ -181,9 +188,32 @@ async def process_message_from_web( # noqa: C901, PLR0912, PLR0915 is_summarize = text == "/summarize" if tg_chat_id and not is_summarize: - await bot.send_message( - tg_chat_id, f"📱 {html.quote(text)}", reply_markup=ReplyKeyboardRemove() - ) + if images_base64 and images_media_types: + if len(images_base64) == 1: + photo_bytes = base64.b64decode(images_base64[0]) + await bot.send_photo( + tg_chat_id, + BufferedInputFile(photo_bytes, "photo.jpg"), + caption=f"📱 {text}" if text else "📱", + reply_markup=ReplyKeyboardRemove(), + ) + else: + media = [] + img_pairs = zip(images_base64, images_media_types, strict=True) + for i, (img_b64, _) in enumerate(img_pairs): + photo_bytes = base64.b64decode(img_b64) + caption = f"📱 {text}" if i == 0 and text else None + media.append( + InputMediaPhoto( + media=BufferedInputFile(photo_bytes, f"photo_{i}.jpg"), + caption=caption, + ) + ) + await bot.send_media_group(tg_chat_id, media) + else: + await bot.send_message( + tg_chat_id, f"📱 {html.quote(text)}", reply_markup=ReplyKeyboardRemove() + ) api_key = user["geminiApiKey"] model_name = user.get("model", "gemini-3-pro-preview") diff --git a/backend/src/bot/sync.py b/backend/src/bot/sync.py index 74dbd26..fed9857 100644 --- a/backend/src/bot/sync.py +++ b/backend/src/bot/sync.py @@ -51,6 +51,8 @@ async def handle_pending_generation(bot: Bot, item: dict, item_id: str) -> None: text=item["userMessage"], bot=bot, convex_chat_id=item["chatId"], + images_base64=item.get("imagesBase64"), + images_media_types=item.get("imagesMediaTypes"), ) except Exception as e: # noqa: BLE001 logger.error(f"Error processing {item_id}: {e}") diff --git a/frontend/.prettierignore b/frontend/.prettierignore index 7d74fe2..04171ab 100644 --- a/frontend/.prettierignore +++ b/frontend/.prettierignore @@ -5,5 +5,8 @@ yarn.lock bun.lock bun.lockb +# Convex generated files +src/lib/convex/_generated/ + # Miscellaneous /static/ diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js index 8e251e7..07f9a32 100644 --- a/frontend/eslint.config.js +++ b/frontend/eslint.config.js @@ -22,9 +22,11 @@ export default defineConfig( languageOptions: { globals: { ...globals.browser, ...globals.node } }, rules: { - // typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects. - // see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors - 'no-undef': 'off' + 'no-undef': 'off', + '@typescript-eslint/no-unused-vars': [ + 'error', + { argsIgnorePattern: '^_', varsIgnorePattern: '^_' } + ] } }, { diff --git a/frontend/src/lib/components/CameraCapture.svelte b/frontend/src/lib/components/CameraCapture.svelte new file mode 100644 index 0000000..c246e38 --- /dev/null +++ b/frontend/src/lib/components/CameraCapture.svelte @@ -0,0 +1,184 @@ + + +
+ {#if error} +
+

{error}

+ +
+ {:else if capturedImage} +
+ +
+
+ + +
+ {:else} + + + + {/if} +
diff --git a/frontend/src/lib/components/CaptureCountdown.svelte b/frontend/src/lib/components/CaptureCountdown.svelte new file mode 100644 index 0000000..562cf4f --- /dev/null +++ b/frontend/src/lib/components/CaptureCountdown.svelte @@ -0,0 +1,144 @@ + + +
+ {#if error} +
+

{error}

+ +
+ {:else} +
+ +
+ {count} +
+
+
+ +
+ {/if} +
diff --git a/frontend/src/lib/components/ChatInput.svelte b/frontend/src/lib/components/ChatInput.svelte index 0bdbde1..06cdc07 100644 --- a/frontend/src/lib/components/ChatInput.svelte +++ b/frontend/src/lib/components/ChatInput.svelte @@ -2,15 +2,16 @@ interface Props { onsubmit: (message: string) => void; disabled?: boolean; + allowEmpty?: boolean; } - let { onsubmit, disabled = false }: Props = $props(); + let { onsubmit, disabled = false, allowEmpty = false }: Props = $props(); let value = $state(''); function handleSubmit(e: Event) { e.preventDefault(); const trimmed = value.trim(); - if (trimmed && !disabled) { + if ((trimmed || allowEmpty) && !disabled) { onsubmit(trimmed); value = ''; } diff --git a/frontend/src/lib/components/DraftBadge.svelte b/frontend/src/lib/components/DraftBadge.svelte new file mode 100644 index 0000000..bac5cda --- /dev/null +++ b/frontend/src/lib/components/DraftBadge.svelte @@ -0,0 +1,27 @@ + + +{#if photos.length > 0} +
+ {#each photos as _photo, i (i)} + + {/each} +
+{/if} diff --git a/frontend/src/lib/components/PhotoButton.svelte b/frontend/src/lib/components/PhotoButton.svelte new file mode 100644 index 0000000..2073ee2 --- /dev/null +++ b/frontend/src/lib/components/PhotoButton.svelte @@ -0,0 +1,65 @@ + + +
+ + + {#if menuOpen} + +
+ + +
+ {/if} +
diff --git a/frontend/src/lib/components/PhotoPreview.svelte b/frontend/src/lib/components/PhotoPreview.svelte new file mode 100644 index 0000000..7813163 --- /dev/null +++ b/frontend/src/lib/components/PhotoPreview.svelte @@ -0,0 +1,27 @@ + + +
+ +
+ diff --git a/frontend/src/lib/components/PhotoRequestPopup.svelte b/frontend/src/lib/components/PhotoRequestPopup.svelte new file mode 100644 index 0000000..5f4a374 --- /dev/null +++ b/frontend/src/lib/components/PhotoRequestPopup.svelte @@ -0,0 +1,22 @@ + + +
+
+

Photo requested

+
+ + +
+
+
diff --git a/frontend/src/lib/components/SilentCapture.svelte b/frontend/src/lib/components/SilentCapture.svelte new file mode 100644 index 0000000..5d8e3e9 --- /dev/null +++ b/frontend/src/lib/components/SilentCapture.svelte @@ -0,0 +1,126 @@ + + +
+ +
diff --git a/frontend/src/lib/components/StealthOverlay.svelte b/frontend/src/lib/components/StealthOverlay.svelte index 88df85f..1d96286 100644 --- a/frontend/src/lib/components/StealthOverlay.svelte +++ b/frontend/src/lib/components/StealthOverlay.svelte @@ -20,6 +20,9 @@ function handleTouchEnd(e: TouchEvent) { if (e.touches.length > 0) return; + const target = e.target as HTMLElement; + if (target?.closest('[data-camera-ui]')) return; + const touch = e.changedTouches[0]; const now = Date.now(); const x = touch.clientX; diff --git a/frontend/src/lib/components/WatchCountdown.svelte b/frontend/src/lib/components/WatchCountdown.svelte new file mode 100644 index 0000000..32a8c37 --- /dev/null +++ b/frontend/src/lib/components/WatchCountdown.svelte @@ -0,0 +1,29 @@ + + +
+ {count} + +
diff --git a/frontend/src/lib/convex-polling.svelte.ts b/frontend/src/lib/convex-polling.svelte.ts index 627e4ae..c6d9464 100644 --- a/frontend/src/lib/convex-polling.svelte.ts +++ b/frontend/src/lib/convex-polling.svelte.ts @@ -3,7 +3,7 @@ import { getContext, setContext } from 'svelte'; import type { FunctionReference, FunctionArgs, FunctionReturnType } from 'convex/server'; const POLLING_CONTEXT_KEY = 'convex-polling'; -const POLL_INTERVAL = 1000; +const POLL_INTERVAL = 500; type PollingContext = { client: ConvexHttpClient; diff --git a/frontend/src/lib/convex/_generated/api.d.ts b/frontend/src/lib/convex/_generated/api.d.ts index 391ba27..90af91a 100644 --- a/frontend/src/lib/convex/_generated/api.d.ts +++ b/frontend/src/lib/convex/_generated/api.d.ts @@ -8,18 +8,30 @@ * @module */ -import type * as chats from '../chats.js'; -import type * as messages from '../messages.js'; -import type * as pendingGenerations from '../pendingGenerations.js'; -import type * as users from '../users.js'; +import type * as chats from "../chats.js"; +import type * as devicePairings from "../devicePairings.js"; +import type * as messages from "../messages.js"; +import type * as pairingRequests from "../pairingRequests.js"; +import type * as pendingGenerations from "../pendingGenerations.js"; +import type * as photoDrafts from "../photoDrafts.js"; +import type * as photoRequests from "../photoRequests.js"; +import type * as users from "../users.js"; -import type { ApiFromModules, FilterApi, FunctionReference } from 'convex/server'; +import type { + ApiFromModules, + FilterApi, + FunctionReference, +} from "convex/server"; declare const fullApi: ApiFromModules<{ - chats: typeof chats; - messages: typeof messages; - pendingGenerations: typeof pendingGenerations; - users: typeof users; + chats: typeof chats; + devicePairings: typeof devicePairings; + messages: typeof messages; + pairingRequests: typeof pairingRequests; + pendingGenerations: typeof pendingGenerations; + photoDrafts: typeof photoDrafts; + photoRequests: typeof photoRequests; + users: typeof users; }>; /** @@ -30,7 +42,10 @@ declare const fullApi: ApiFromModules<{ * const myFunctionReference = api.myModule.myFunction; * ``` */ -export declare const api: FilterApi>; +export declare const api: FilterApi< + typeof fullApi, + FunctionReference +>; /** * A utility for referencing Convex functions in your app's internal API. @@ -40,6 +55,9 @@ export declare const api: FilterApi>; +export declare const internal: FilterApi< + typeof fullApi, + FunctionReference +>; export declare const components: {}; diff --git a/frontend/src/lib/convex/_generated/api.js b/frontend/src/lib/convex/_generated/api.js index 24593c7..44bf985 100644 --- a/frontend/src/lib/convex/_generated/api.js +++ b/frontend/src/lib/convex/_generated/api.js @@ -8,7 +8,7 @@ * @module */ -import { anyApi, componentsGeneric } from 'convex/server'; +import { anyApi, componentsGeneric } from "convex/server"; /** * A utility for referencing Convex functions in your app's API. diff --git a/frontend/src/lib/convex/_generated/dataModel.d.ts b/frontend/src/lib/convex/_generated/dataModel.d.ts index 5428df6..f97fd19 100644 --- a/frontend/src/lib/convex/_generated/dataModel.d.ts +++ b/frontend/src/lib/convex/_generated/dataModel.d.ts @@ -9,13 +9,13 @@ */ import type { - DataModelFromSchemaDefinition, - DocumentByName, - TableNamesInDataModel, - SystemTableNames -} from 'convex/server'; -import type { GenericId } from 'convex/values'; -import schema from '../schema.js'; + DataModelFromSchemaDefinition, + DocumentByName, + TableNamesInDataModel, + SystemTableNames, +} from "convex/server"; +import type { GenericId } from "convex/values"; +import schema from "../schema.js"; /** * The names of all of your Convex tables. @@ -27,7 +27,10 @@ export type TableNames = TableNamesInDataModel; * * @typeParam TableName - A string literal type of the table name (like "users"). */ -export type Doc = DocumentByName; +export type Doc = DocumentByName< + DataModel, + TableName +>; /** * An identifier for a document in Convex. @@ -42,7 +45,8 @@ export type Doc = DocumentByName = GenericId; +export type Id = + GenericId; /** * A type describing your Convex data model. diff --git a/frontend/src/lib/convex/_generated/server.d.ts b/frontend/src/lib/convex/_generated/server.d.ts index 1cc047e..bec05e6 100644 --- a/frontend/src/lib/convex/_generated/server.d.ts +++ b/frontend/src/lib/convex/_generated/server.d.ts @@ -9,17 +9,17 @@ */ import { - ActionBuilder, - HttpActionBuilder, - MutationBuilder, - QueryBuilder, - GenericActionCtx, - GenericMutationCtx, - GenericQueryCtx, - GenericDatabaseReader, - GenericDatabaseWriter -} from 'convex/server'; -import type { DataModel } from './dataModel.js'; + ActionBuilder, + HttpActionBuilder, + MutationBuilder, + QueryBuilder, + GenericActionCtx, + GenericMutationCtx, + GenericQueryCtx, + GenericDatabaseReader, + GenericDatabaseWriter, +} from "convex/server"; +import type { DataModel } from "./dataModel.js"; /** * Define a query in this Convex app's public API. @@ -29,7 +29,7 @@ import type { DataModel } from './dataModel.js'; * @param func - The query function. It receives a {@link QueryCtx} as its first argument. * @returns The wrapped query. Include this as an `export` to name it and make it accessible. */ -export declare const query: QueryBuilder; +export declare const query: QueryBuilder; /** * Define a query that is only accessible from other Convex functions (but not from the client). @@ -39,7 +39,7 @@ export declare const query: QueryBuilder; * @param func - The query function. It receives a {@link QueryCtx} as its first argument. * @returns The wrapped query. Include this as an `export` to name it and make it accessible. */ -export declare const internalQuery: QueryBuilder; +export declare const internalQuery: QueryBuilder; /** * Define a mutation in this Convex app's public API. @@ -49,7 +49,7 @@ export declare const internalQuery: QueryBuilder; * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. */ -export declare const mutation: MutationBuilder; +export declare const mutation: MutationBuilder; /** * Define a mutation that is only accessible from other Convex functions (but not from the client). @@ -59,7 +59,7 @@ export declare const mutation: MutationBuilder; * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. */ -export declare const internalMutation: MutationBuilder; +export declare const internalMutation: MutationBuilder; /** * Define an action in this Convex app's public API. @@ -72,7 +72,7 @@ export declare const internalMutation: MutationBuilder; * @param func - The action. It receives an {@link ActionCtx} as its first argument. * @returns The wrapped action. Include this as an `export` to name it and make it accessible. */ -export declare const action: ActionBuilder; +export declare const action: ActionBuilder; /** * Define an action that is only accessible from other Convex functions (but not from the client). @@ -80,7 +80,7 @@ export declare const action: ActionBuilder; * @param func - The function. It receives an {@link ActionCtx} as its first argument. * @returns The wrapped function. Include this as an `export` to name it and make it accessible. */ -export declare const internalAction: ActionBuilder; +export declare const internalAction: ActionBuilder; /** * Define an HTTP action. diff --git a/frontend/src/lib/convex/_generated/server.js b/frontend/src/lib/convex/_generated/server.js index a18aa28..bf3d25a 100644 --- a/frontend/src/lib/convex/_generated/server.js +++ b/frontend/src/lib/convex/_generated/server.js @@ -9,14 +9,14 @@ */ import { - actionGeneric, - httpActionGeneric, - queryGeneric, - mutationGeneric, - internalActionGeneric, - internalMutationGeneric, - internalQueryGeneric -} from 'convex/server'; + actionGeneric, + httpActionGeneric, + queryGeneric, + mutationGeneric, + internalActionGeneric, + internalMutationGeneric, + internalQueryGeneric, +} from "convex/server"; /** * Define a query in this Convex app's public API. diff --git a/frontend/src/lib/convex/chats.ts b/frontend/src/lib/convex/chats.ts index 51430ca..7bd825c 100644 --- a/frontend/src/lib/convex/chats.ts +++ b/frontend/src/lib/convex/chats.ts @@ -43,8 +43,15 @@ export const clear = mutation({ .collect(); for (const message of messages) { - if (args.preserveImages && message.imageBase64) { - continue; + if (args.preserveImages) { + const hasLegacyImage = message.imageBase64 || message.imagesBase64?.length; + const messageImages = await ctx.db + .query('messageImages') + .withIndex('by_message_id', (q) => q.eq('messageId', message._id)) + .first(); + if (hasLegacyImage || messageImages) { + continue; + } } await ctx.db.delete(message._id); } diff --git a/frontend/src/lib/convex/devicePairings.ts b/frontend/src/lib/convex/devicePairings.ts new file mode 100644 index 0000000..02c4f22 --- /dev/null +++ b/frontend/src/lib/convex/devicePairings.ts @@ -0,0 +1,109 @@ +import { v } from 'convex/values'; +import { mutation, query } from './_generated/server'; + +export const register = mutation({ + args: { + chatId: v.id('chats'), + deviceId: v.string(), + hasCamera: v.boolean() + }, + returns: v.id('devicePairings'), + handler: async (ctx, args) => { + const existing = await ctx.db + .query('devicePairings') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .collect(); + + const device = existing.find((d) => d.deviceId === args.deviceId); + + if (device) { + await ctx.db.patch(device._id, { + hasCamera: args.hasCamera, + lastSeen: Date.now() + }); + return device._id; + } + + return await ctx.db.insert('devicePairings', { + chatId: args.chatId, + deviceId: args.deviceId, + hasCamera: args.hasCamera, + lastSeen: Date.now() + }); + } +}); + +export const heartbeat = mutation({ + args: { + chatId: v.id('chats'), + deviceId: v.string() + }, + returns: v.null(), + handler: async (ctx, args) => { + const devices = await ctx.db + .query('devicePairings') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .collect(); + + const device = devices.find((d) => d.deviceId === args.deviceId); + if (device) { + await ctx.db.patch(device._id, { lastSeen: Date.now() }); + } + return null; + } +}); + +export const getMyDevice = query({ + args: { chatId: v.id('chats'), deviceId: v.string() }, + returns: v.union( + v.object({ + _id: v.id('devicePairings'), + _creationTime: v.number(), + chatId: v.id('chats'), + deviceId: v.string(), + hasCamera: v.boolean(), + pairedWithDeviceId: v.optional(v.string()), + lastSeen: v.number() + }), + v.null() + ), + handler: async (ctx, args) => { + const devices = await ctx.db + .query('devicePairings') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .collect(); + + return devices.find((d) => d.deviceId === args.deviceId) ?? null; + } +}); + +export const getPairedDevice = query({ + args: { chatId: v.id('chats'), deviceId: v.string() }, + returns: v.union( + v.object({ + _id: v.id('devicePairings'), + _creationTime: v.number(), + chatId: v.id('chats'), + deviceId: v.string(), + hasCamera: v.boolean(), + pairedWithDeviceId: v.optional(v.string()), + lastSeen: v.number() + }), + v.null() + ), + handler: async (ctx, args) => { + const devices = await ctx.db + .query('devicePairings') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .collect(); + + const myDevice = devices.find((d) => d.deviceId === args.deviceId); + if (!myDevice?.pairedWithDeviceId) return null; + + const thirtySecondsAgo = Date.now() - 30000; + const paired = devices.find( + (d) => d.deviceId === myDevice.pairedWithDeviceId && d.lastSeen > thirtySecondsAgo + ); + return paired ?? null; + } +}); diff --git a/frontend/src/lib/convex/messages.ts b/frontend/src/lib/convex/messages.ts index f268037..f0811b1 100644 --- a/frontend/src/lib/convex/messages.ts +++ b/frontend/src/lib/convex/messages.ts @@ -1,5 +1,6 @@ import { v } from 'convex/values'; import { mutation, query } from './_generated/server'; +import type { Id } from './_generated/dataModel'; export const listByChat = query({ args: { chatId: v.id('chats') }, @@ -39,6 +40,7 @@ export const create = mutation({ imageMediaType: v.optional(v.string()), imagesBase64: v.optional(v.array(v.string())), imagesMediaTypes: v.optional(v.array(v.string())), + photoDraftIds: v.optional(v.array(v.id('photoDrafts'))), followUpOptions: v.optional(v.array(v.string())), isStreaming: v.optional(v.boolean()) }, @@ -58,18 +60,50 @@ export const create = mutation({ isStreaming: args.isStreaming }); + const drafts: Array<{ base64: string; mediaType: string; id: Id<'photoDrafts'> }> = []; + if (args.photoDraftIds && args.photoDraftIds.length > 0) { + for (const draftId of args.photoDraftIds) { + const draft = await ctx.db.get(draftId); + if (draft) { + drafts.push({ base64: draft.base64, mediaType: draft.mediaType, id: draft._id }); + } + } + } + + for (let i = 0; i < drafts.length; i++) { + await ctx.db.insert('messageImages', { + messageId, + base64: drafts[i].base64, + mediaType: drafts[i].mediaType, + order: i + }); + } + if (args.source === 'web' && args.role === 'user') { const chat = await ctx.db.get(args.chatId); if (chat) { - await ctx.db.insert('pendingGenerations', { + const pendingGenId = await ctx.db.insert('pendingGenerations', { userId: chat.userId, chatId: args.chatId, userMessage: args.content, createdAt: Date.now() }); + + for (let i = 0; i < drafts.length; i++) { + await ctx.db.insert('pendingGenerationImages', { + pendingGenerationId: pendingGenId, + base64: drafts[i].base64, + mediaType: drafts[i].mediaType, + order: i + }); + } } } + for (const draft of drafts) { + await ctx.db.delete(draft.id); + } + return messageId; } }); @@ -177,6 +211,15 @@ export const getChatImages = query({ const images: Array<{ base64: string; mediaType: string }> = []; for (const m of messages) { + const msgImages = await ctx.db + .query('messageImages') + .withIndex('by_message_id', (q) => q.eq('messageId', m._id)) + .collect(); + + for (const img of msgImages.sort((a, b) => a.order - b.order)) { + images.push({ base64: img.base64, mediaType: img.mediaType }); + } + if (m.imagesBase64 && m.imagesMediaTypes) { for (let i = 0; i < m.imagesBase64.length; i++) { images.push({ diff --git a/frontend/src/lib/convex/pairingRequests.ts b/frontend/src/lib/convex/pairingRequests.ts new file mode 100644 index 0000000..7270a93 --- /dev/null +++ b/frontend/src/lib/convex/pairingRequests.ts @@ -0,0 +1,122 @@ +import { v } from 'convex/values'; +import { mutation, query } from './_generated/server'; + +export const create = mutation({ + args: { + chatId: v.id('chats'), + fromDeviceId: v.string() + }, + returns: v.id('pairingRequests'), + handler: async (ctx, args) => { + const existing = await ctx.db + .query('pairingRequests') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .collect(); + + const pending = existing.find( + (r) => r.fromDeviceId === args.fromDeviceId && r.status === 'pending' + ); + if (pending) return pending._id; + + return await ctx.db.insert('pairingRequests', { + chatId: args.chatId, + fromDeviceId: args.fromDeviceId, + status: 'pending', + createdAt: Date.now() + }); + } +}); + +export const accept = mutation({ + args: { + requestId: v.id('pairingRequests'), + acceptingDeviceId: v.string() + }, + returns: v.null(), + handler: async (ctx, args) => { + const request = await ctx.db.get(args.requestId); + if (!request || request.status !== 'pending') return null; + + await ctx.db.patch(args.requestId, { status: 'accepted' }); + + const devices = await ctx.db + .query('devicePairings') + .withIndex('by_chat_id', (q) => q.eq('chatId', request.chatId)) + .collect(); + + const fromDevice = devices.find((d) => d.deviceId === request.fromDeviceId); + const acceptingDevice = devices.find((d) => d.deviceId === args.acceptingDeviceId); + + if (fromDevice) { + await ctx.db.patch(fromDevice._id, { pairedWithDeviceId: args.acceptingDeviceId }); + } + if (acceptingDevice) { + await ctx.db.patch(acceptingDevice._id, { pairedWithDeviceId: request.fromDeviceId }); + } + + return null; + } +}); + +export const reject = mutation({ + args: { requestId: v.id('pairingRequests') }, + returns: v.null(), + handler: async (ctx, args) => { + const request = await ctx.db.get(args.requestId); + if (!request || request.status !== 'pending') return null; + await ctx.db.patch(args.requestId, { status: 'rejected' }); + return null; + } +}); + +export const getPending = query({ + args: { chatId: v.id('chats'), excludeDeviceId: v.string() }, + returns: v.union( + v.object({ + _id: v.id('pairingRequests'), + _creationTime: v.number(), + chatId: v.id('chats'), + fromDeviceId: v.string(), + status: v.union(v.literal('pending'), v.literal('accepted'), v.literal('rejected')), + createdAt: v.number() + }), + v.null() + ), + handler: async (ctx, args) => { + const requests = await ctx.db + .query('pairingRequests') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .collect(); + + return ( + requests.find((r) => r.status === 'pending' && r.fromDeviceId !== args.excludeDeviceId) ?? + null + ); + } +}); + +export const unpair = mutation({ + args: { + chatId: v.id('chats'), + deviceId: v.string() + }, + returns: v.null(), + handler: async (ctx, args) => { + const devices = await ctx.db + .query('devicePairings') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .collect(); + + const myDevice = devices.find((d) => d.deviceId === args.deviceId); + if (!myDevice?.pairedWithDeviceId) return null; + + const pairedDevice = devices.find((d) => d.deviceId === myDevice.pairedWithDeviceId); + + await ctx.db.patch(myDevice._id, { pairedWithDeviceId: undefined }); + if (pairedDevice) { + await ctx.db.patch(pairedDevice._id, { pairedWithDeviceId: undefined }); + } + + return null; + } +}); diff --git a/frontend/src/lib/convex/pendingGenerations.ts b/frontend/src/lib/convex/pendingGenerations.ts index 47be6d9..db4c8e5 100644 --- a/frontend/src/lib/convex/pendingGenerations.ts +++ b/frontend/src/lib/convex/pendingGenerations.ts @@ -10,11 +10,33 @@ export const list = query({ userId: v.id('users'), chatId: v.id('chats'), userMessage: v.string(), + imagesBase64: v.optional(v.array(v.string())), + imagesMediaTypes: v.optional(v.array(v.string())), createdAt: v.number() }) ), handler: async (ctx) => { - return await ctx.db.query('pendingGenerations').collect(); + const pending = await ctx.db.query('pendingGenerations').collect(); + + const result = []; + for (const p of pending) { + const images = await ctx.db + .query('pendingGenerationImages') + .withIndex('by_pending_generation_id', (q) => q.eq('pendingGenerationId', p._id)) + .collect(); + + const sortedImages = images.sort((a, b) => a.order - b.order); + + result.push({ + ...p, + imagesBase64: + sortedImages.length > 0 ? sortedImages.map((img) => img.base64) : p.imagesBase64, + imagesMediaTypes: + sortedImages.length > 0 ? sortedImages.map((img) => img.mediaType) : p.imagesMediaTypes + }); + } + + return result; } }); @@ -39,7 +61,35 @@ export const remove = mutation({ args: { id: v.id('pendingGenerations') }, returns: v.null(), handler: async (ctx, args) => { + const images = await ctx.db + .query('pendingGenerationImages') + .withIndex('by_pending_generation_id', (q) => q.eq('pendingGenerationId', args.id)) + .collect(); + for (const img of images) { + await ctx.db.delete(img._id); + } await ctx.db.delete(args.id); return null; } }); + +export const getImages = query({ + args: { pendingGenerationId: v.id('pendingGenerations') }, + returns: v.array( + v.object({ + base64: v.string(), + mediaType: v.string() + }) + ), + handler: async (ctx, args) => { + const images = await ctx.db + .query('pendingGenerationImages') + .withIndex('by_pending_generation_id', (q) => + q.eq('pendingGenerationId', args.pendingGenerationId) + ) + .collect(); + return images + .sort((a, b) => a.order - b.order) + .map((img) => ({ base64: img.base64, mediaType: img.mediaType })); + } +}); diff --git a/frontend/src/lib/convex/photoDrafts.ts b/frontend/src/lib/convex/photoDrafts.ts new file mode 100644 index 0000000..ea5b937 --- /dev/null +++ b/frontend/src/lib/convex/photoDrafts.ts @@ -0,0 +1,95 @@ +import { v } from 'convex/values'; +import { mutation, query } from './_generated/server'; + +const photoValidator = v.object({ + base64: v.string(), + mediaType: v.string() +}); + +export const get = query({ + args: { chatId: v.id('chats'), deviceId: v.string() }, + returns: v.object({ + photos: v.array( + v.object({ + _id: v.id('photoDrafts'), + base64: v.string(), + mediaType: v.string() + }) + ) + }), + handler: async (ctx, args) => { + const drafts = await ctx.db + .query('photoDrafts') + .withIndex('by_chat_id_and_device_id', (q) => + q.eq('chatId', args.chatId).eq('deviceId', args.deviceId) + ) + .collect(); + + return { + photos: drafts.map((d) => ({ + _id: d._id, + base64: d.base64, + mediaType: d.mediaType + })) + }; + } +}); + +export const addPhoto = mutation({ + args: { + chatId: v.id('chats'), + deviceId: v.string(), + photo: photoValidator + }, + returns: v.null(), + handler: async (ctx, args) => { + await ctx.db.insert('photoDrafts', { + chatId: args.chatId, + deviceId: args.deviceId, + base64: args.photo.base64, + mediaType: args.photo.mediaType, + createdAt: Date.now() + }); + return null; + } +}); + +export const removePhoto = mutation({ + args: { + chatId: v.id('chats'), + deviceId: v.string(), + index: v.number() + }, + returns: v.null(), + handler: async (ctx, args) => { + const drafts = await ctx.db + .query('photoDrafts') + .withIndex('by_chat_id_and_device_id', (q) => + q.eq('chatId', args.chatId).eq('deviceId', args.deviceId) + ) + .collect(); + + if (drafts[args.index]) { + await ctx.db.delete(drafts[args.index]._id); + } + return null; + } +}); + +export const clear = mutation({ + args: { chatId: v.id('chats'), deviceId: v.string() }, + returns: v.null(), + handler: async (ctx, args) => { + const drafts = await ctx.db + .query('photoDrafts') + .withIndex('by_chat_id_and_device_id', (q) => + q.eq('chatId', args.chatId).eq('deviceId', args.deviceId) + ) + .collect(); + + for (const draft of drafts) { + await ctx.db.delete(draft._id); + } + return null; + } +}); diff --git a/frontend/src/lib/convex/photoRequests.ts b/frontend/src/lib/convex/photoRequests.ts new file mode 100644 index 0000000..5a537f1 --- /dev/null +++ b/frontend/src/lib/convex/photoRequests.ts @@ -0,0 +1,171 @@ +import { v } from 'convex/values'; +import { mutation, query } from './_generated/server'; + +const photoRequestValidator = v.object({ + _id: v.id('photoRequests'), + _creationTime: v.number(), + chatId: v.id('chats'), + requesterId: v.string(), + captureDeviceId: v.optional(v.string()), + status: v.union( + v.literal('pending'), + v.literal('countdown'), + v.literal('capture_now'), + v.literal('captured'), + v.literal('accepted'), + v.literal('rejected') + ), + photoBase64: v.optional(v.string()), + photoMediaType: v.optional(v.string()), + thumbnailBase64: v.optional(v.string()), + createdAt: v.number() +}); + +export const create = mutation({ + args: { + chatId: v.id('chats'), + requesterId: v.string(), + captureDeviceId: v.string() + }, + returns: v.id('photoRequests'), + handler: async (ctx, args) => { + return await ctx.db.insert('photoRequests', { + chatId: args.chatId, + requesterId: args.requesterId, + captureDeviceId: args.captureDeviceId, + status: 'countdown', + createdAt: Date.now() + }); + } +}); + +export const markCaptureNow = mutation({ + args: { requestId: v.id('photoRequests') }, + returns: v.null(), + handler: async (ctx, args) => { + await ctx.db.patch(args.requestId, { status: 'capture_now' }); + return null; + } +}); + +export const submitPhoto = mutation({ + args: { + requestId: v.id('photoRequests'), + photoBase64: v.string(), + photoMediaType: v.string(), + thumbnailBase64: v.optional(v.string()) + }, + returns: v.null(), + handler: async (ctx, args) => { + await ctx.db.patch(args.requestId, { + status: 'captured', + photoBase64: args.photoBase64, + photoMediaType: args.photoMediaType, + thumbnailBase64: args.thumbnailBase64 + }); + return null; + } +}); + +export const markAccepted = mutation({ + args: { requestId: v.id('photoRequests') }, + returns: v.null(), + handler: async (ctx, args) => { + await ctx.db.patch(args.requestId, { status: 'accepted' }); + return null; + } +}); + +export const markRejected = mutation({ + args: { requestId: v.id('photoRequests') }, + returns: v.null(), + handler: async (ctx, args) => { + await ctx.db.patch(args.requestId, { status: 'rejected' }); + return null; + } +}); + +export const getCaptureNowRequest = query({ + args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) }, + returns: v.union(photoRequestValidator, v.null()), + handler: async (ctx, args) => { + const requests = await ctx.db + .query('photoRequests') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .order('desc') + .take(50); + + return requests.find((r) => r.status === 'capture_now') ?? null; + } +}); + +export const getActiveForCapture = query({ + args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) }, + returns: v.union(photoRequestValidator, v.null()), + handler: async (ctx, args) => { + const requests = await ctx.db + .query('photoRequests') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .order('desc') + .take(50); + + return requests.find((r) => r.status === 'countdown' || r.status === 'capture_now') ?? null; + } +}); + +export const getMyActiveRequest = query({ + args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) }, + returns: v.union(photoRequestValidator, v.null()), + handler: async (ctx, args) => { + const requests = await ctx.db + .query('photoRequests') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .take(100); + + if (!args.deviceId) return null; + + return ( + requests.find( + (r) => + r.requesterId === args.deviceId && + (r.status === 'countdown' || r.status === 'capture_now' || r.status === 'captured') + ) ?? null + ); + } +}); + +export const cleanup = mutation({ + args: { chatId: v.id('chats') }, + returns: v.number(), + handler: async (ctx, args) => { + const requests = await ctx.db + .query('photoRequests') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .take(20); + + let deleted = 0; + for (const req of requests) { + await ctx.db.delete(req._id); + deleted++; + } + return deleted; + } +}); + +export const getCapturedForPhone = query({ + args: { chatId: v.id('chats'), deviceId: v.optional(v.string()) }, + returns: v.union(photoRequestValidator, v.null()), + handler: async (ctx, args) => { + const requests = await ctx.db + .query('photoRequests') + .withIndex('by_chat_id', (q) => q.eq('chatId', args.chatId)) + .order('desc') + .take(50); + + if (!args.deviceId) return null; + + return ( + requests.find((r) => r.captureDeviceId === args.deviceId && r.status === 'captured') ?? null + ); + } +}); diff --git a/frontend/src/lib/convex/schema.ts b/frontend/src/lib/convex/schema.ts index a199fc7..98b3734 100644 --- a/frontend/src/lib/convex/schema.ts +++ b/frontend/src/lib/convex/schema.ts @@ -40,6 +40,63 @@ export default defineSchema({ userId: v.id('users'), chatId: v.id('chats'), userMessage: v.string(), + imagesBase64: v.optional(v.array(v.string())), + imagesMediaTypes: v.optional(v.array(v.string())), createdAt: v.number() - }) + }), + + pendingGenerationImages: defineTable({ + pendingGenerationId: v.id('pendingGenerations'), + base64: v.string(), + mediaType: v.string(), + order: v.number() + }).index('by_pending_generation_id', ['pendingGenerationId']), + + messageImages: defineTable({ + messageId: v.id('messages'), + base64: v.string(), + mediaType: v.string(), + order: v.number() + }).index('by_message_id', ['messageId']), + + devicePairings: defineTable({ + chatId: v.id('chats'), + deviceId: v.string(), + hasCamera: v.boolean(), + pairedWithDeviceId: v.optional(v.string()), + lastSeen: v.number() + }).index('by_chat_id', ['chatId']), + + pairingRequests: defineTable({ + chatId: v.id('chats'), + fromDeviceId: v.string(), + status: v.union(v.literal('pending'), v.literal('accepted'), v.literal('rejected')), + createdAt: v.number() + }).index('by_chat_id', ['chatId']), + + photoRequests: defineTable({ + chatId: v.id('chats'), + requesterId: v.string(), + captureDeviceId: v.optional(v.string()), + status: v.union( + v.literal('pending'), + v.literal('countdown'), + v.literal('capture_now'), + v.literal('captured'), + v.literal('accepted'), + v.literal('rejected') + ), + photoBase64: v.optional(v.string()), + photoMediaType: v.optional(v.string()), + thumbnailBase64: v.optional(v.string()), + createdAt: v.number() + }).index('by_chat_id', ['chatId']), + + photoDrafts: defineTable({ + chatId: v.id('chats'), + deviceId: v.string(), + base64: v.string(), + mediaType: v.string(), + createdAt: v.number() + }).index('by_chat_id_and_device_id', ['chatId', 'deviceId']) }); diff --git a/frontend/src/routes/[mnemonic]/+page.svelte b/frontend/src/routes/[mnemonic]/+page.svelte index cfd4940..1cb9487 100644 --- a/frontend/src/routes/[mnemonic]/+page.svelte +++ b/frontend/src/routes/[mnemonic]/+page.svelte @@ -1,13 +1,21 @@