feat: wire image uploads into AI assistant chat (vision support)

- Backend: ChatMessageRequest accepts upload_ids, endpoint fetches
  images from S3, base64-encodes them, passes to Claude as multimodal
  content blocks (vision API)
- Backend: add download_file() to storage_service for fetching from S3
- Frontend: handleSend collects completed upload IDs from pendingUploads
  and includes them in the sendChatMessage API call
- Frontend: prefill handler passes upload IDs from dashboard nav state
- Enables paste-screenshot → AI-sees-it flow end-to-end

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
chihlasm
2026-03-24 04:39:54 +00:00
parent 48f2b3faaf
commit 3b682069d3
6 changed files with 99 additions and 7 deletions

View File

@@ -55,7 +55,9 @@ export default function AssistantChatPage() {
// Handle prefill from command palette / dashboard handoff
useEffect(() => {
const prefill = (location.state as { prefill?: string } | null)?.prefill
const state = location.state as { prefill?: string; uploadIds?: string[] } | null
const prefill = state?.prefill
const uploadIds = state?.uploadIds
if (!prefill || prefillHandledRef.current) return
prefillHandledRef.current = true
@@ -80,7 +82,10 @@ export default function AssistantChatPage() {
setMessages([{ role: 'user', content: prefill }])
setLoading(true)
const response = await aiSessionsApi.sendChatMessage(session.session_id, { message: prefill })
const response = await aiSessionsApi.sendChatMessage(session.session_id, {
message: prefill,
upload_ids: uploadIds?.length ? uploadIds : undefined,
})
setMessages(prev => [
...prev,
{ role: 'assistant', content: response.content, suggestedFlows: response.suggested_flows },
@@ -183,12 +188,19 @@ export default function AssistantChatPage() {
if (!input.trim() || !activeChatId || loading) return
const userMessage = input.trim()
const completedUploadIds = pendingUploads
.filter((u) => u.status === 'done' && u.result?.id)
.map((u) => u.result!.id)
setInput('')
setPendingUploads([])
setMessages(prev => [...prev, { role: 'user', content: userMessage }])
setLoading(true)
try {
const response = await aiSessionsApi.sendChatMessage(activeChatId, { message: userMessage })
const response = await aiSessionsApi.sendChatMessage(activeChatId, {
message: userMessage,
upload_ids: completedUploadIds.length > 0 ? completedUploadIds : undefined,
})
analytics.aiFeatureUsed({ feature: 'assistant_chat' })
setMessages(prev => [
...prev,

View File

@@ -216,6 +216,7 @@ export interface ChatSessionCreateResponse {
export interface ChatMessageRequest {
message: string
upload_ids?: string[]
}
export interface ChatMessageResponse {