From 176fa649885b96ade7eb4070edc20a457cb11a5b Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan Date: Thu, 14 May 2026 17:36:42 -0700 Subject: [PATCH 1/6] File block v4 --- .../integrations/data/icon-mapping.ts | 1 + .../integrations/data/integrations.json | 14 +- apps/sim/app/api/files/parse/route.test.ts | 34 +++ apps/sim/app/api/files/parse/route.ts | 24 +- apps/sim/app/api/tools/file/manage/route.ts | 149 ++++++++- apps/sim/blocks/blocks.test.ts | 69 +++++ apps/sim/blocks/blocks/file.ts | 289 ++++++++++++++++++ apps/sim/blocks/registry.ts | 3 +- .../sim/lib/api/contracts/storage-transfer.ts | 1 + apps/sim/lib/api/contracts/tools/file.ts | 12 + apps/sim/tools/file/get.ts | 64 ++++ apps/sim/tools/file/index.ts | 10 +- apps/sim/tools/file/parser.ts | 77 +++-- apps/sim/tools/file/types.ts | 3 +- apps/sim/tools/registry.ts | 4 + 15 files changed, 712 insertions(+), 42 deletions(-) diff --git a/apps/sim/app/(landing)/integrations/data/icon-mapping.ts b/apps/sim/app/(landing)/integrations/data/icon-mapping.ts index f80e850cebe..c4b0dfba957 100644 --- a/apps/sim/app/(landing)/integrations/data/icon-mapping.ts +++ b/apps/sim/app/(landing)/integrations/data/icon-mapping.ts @@ -258,6 +258,7 @@ export const blockTypeToIconMap: Record = { extend_v2: ExtendIcon, fathom: FathomIcon, file_v3: DocumentIcon, + file_v4: DocumentIcon, firecrawl: FirecrawlIcon, fireflies_v2: FirefliesIcon, gamma: GammaIcon, diff --git a/apps/sim/app/(landing)/integrations/data/integrations.json b/apps/sim/app/(landing)/integrations/data/integrations.json index 3e6de1c0eef..0ba46960d75 100644 --- a/apps/sim/app/(landing)/integrations/data/integrations.json +++ b/apps/sim/app/(landing)/integrations/data/integrations.json @@ -4032,18 +4032,22 @@ "tags": ["meeting", "note-taking"] }, { - "type": "file_v3", + "type": "file_v4", "slug": "file", "name": "File", - "description": "Read and write workspace files", - "longDescription": "Read and parse files from uploads or URLs, write new workspace files, or append content to existing files.", + "description": "Read, fetch, write, and append files", + "longDescription": "Read workspace files by picker or canonical ID, fetch and parse files from URLs with optional headers, write new workspace files, or append content to existing files.", "bgColor": "#40916C", "iconName": "DocumentIcon", "docsUrl": "https://docs.sim.ai/tools/file", "operations": [ { "name": "Read", - "description": "Parse one or more uploaded files or files from URLs (text, PDF, CSV, images, etc.)" + "description": "Get a workspace file object from a selected file or canonical workspace file ID." + }, + { + "name": "Fetch", + "description": "Parse a file from a URL with optional custom headers for authenticated downloads." }, { "name": "Write", @@ -4054,7 +4058,7 @@ "description": "Append content to an existing workspace file. The file must already exist. Content is added to the end of the file." } ], - "operationCount": 3, + "operationCount": 4, "triggers": [], "triggerCount": 0, "authType": "none", diff --git a/apps/sim/app/api/files/parse/route.test.ts b/apps/sim/app/api/files/parse/route.test.ts index 4283b6723b8..8c18422bae3 100644 --- a/apps/sim/app/api/files/parse/route.test.ts +++ b/apps/sim/app/api/files/parse/route.test.ts @@ -8,6 +8,7 @@ import { createMockRequest, hybridAuthMockFns, inputValidationMock, + inputValidationMockFns, permissionsMock, permissionsMockFns, storageServiceMock, @@ -310,6 +311,39 @@ describe('File Parse API Route', () => { expect(data.results).toHaveLength(2) }) + it('should pass custom headers when fetching external URLs', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('private file content', { + status: 200, + headers: { 'content-type': 'text/plain' }, + }) + ) + + const headers = { Authorization: 'Bearer xoxb-test-token' } + const req = createMockRequest('POST', { + filePath: 'https://files.slack.com/files-pri/T000-F000/download/report.txt', + headers, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( + 'https://files.slack.com/files-pri/T000-F000/download/report.txt', + '203.0.113.10', + expect.objectContaining({ + timeout: 30000, + headers, + }) + ) + }) + it('should process execution file URLs with context query param', async () => { setupFileApiMocks({ cloudEnabled: true, diff --git a/apps/sim/app/api/files/parse/route.ts b/apps/sim/app/api/files/parse/route.ts index e52618c53fa..4bcd7d01914 100644 --- a/apps/sim/app/api/files/parse/route.ts +++ b/apps/sim/app/api/files/parse/route.ts @@ -110,7 +110,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) if (!parsed.success) return parsed.response - const { filePath, fileType, workspaceId, workflowId, executionId } = parsed.data.body + const { filePath, fileType, headers, workspaceId, workflowId, executionId } = parsed.data.body if (!filePath || (typeof filePath === 'string' && filePath.trim() === '')) { return NextResponse.json({ success: false, error: 'No file path provided' }, { status: 400 }) @@ -128,6 +128,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { workspaceId, userId, hasExecutionContext: !!executionContext, + hasHeaders: Boolean(headers && Object.keys(headers).length > 0), }) if (Array.isArray(filePath)) { @@ -146,7 +147,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { fileType, workspaceId, userId, - executionContext + executionContext, + headers ) if (result.metadata) { result.metadata.processingTime = Date.now() - startTime @@ -180,7 +182,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }) } - const result = await parseFileSingle(filePath, fileType, workspaceId, userId, executionContext) + const result = await parseFileSingle( + filePath, + fileType, + workspaceId, + userId, + executionContext, + headers + ) if (result.metadata) { result.metadata.processingTime = Date.now() - startTime @@ -225,7 +234,8 @@ async function parseFileSingle( fileType: string, workspaceId: string, userId: string, - executionContext?: ExecutionContext + executionContext?: ExecutionContext, + headers?: Record ): Promise { logger.info('Parsing file:', filePath) @@ -251,7 +261,7 @@ async function parseFileSingle( } if (filePath.startsWith('http://') || filePath.startsWith('https://')) { - return handleExternalUrl(filePath, fileType, workspaceId, userId, executionContext) + return handleExternalUrl(filePath, fileType, workspaceId, userId, executionContext, headers) } if (isUsingCloudStorage()) { @@ -298,7 +308,8 @@ async function handleExternalUrl( fileType: string, workspaceId: string, userId: string, - executionContext?: ExecutionContext + executionContext?: ExecutionContext, + headers?: Record ): Promise { try { logger.info('Fetching external URL:', url) @@ -382,6 +393,7 @@ async function handleExternalUrl( const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { timeout: DOWNLOAD_TIMEOUT_MS, + ...(headers && Object.keys(headers).length > 0 && { headers }), }) if (!response.ok) { throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`) diff --git a/apps/sim/app/api/tools/file/manage/route.ts b/apps/sim/app/api/tools/file/manage/route.ts index 60082411473..65c129787b1 100644 --- a/apps/sim/app/api/tools/file/manage/route.ts +++ b/apps/sim/app/api/tools/file/manage/route.ts @@ -20,6 +20,100 @@ export const dynamic = 'force-dynamic' const logger = createLogger('FileManageAPI') +const workspaceFileToUserFile = (file: Awaited>) => { + if (!file) return null + + return { + id: file.id, + name: file.name, + url: ensureAbsoluteUrl(file.path), + size: file.size, + type: file.type, + key: file.key, + context: 'workspace', + } +} + +const fileInputToUserFile = (fileInput: unknown) => { + if (!fileInput || typeof fileInput !== 'object' || Array.isArray(fileInput)) return null + + const record = fileInput as Record + const id = + typeof record.id === 'string' + ? record.id.trim() + : typeof record.fileId === 'string' + ? record.fileId.trim() + : '' + + // Objects with ids are resolved through workspace metadata. This fallback is for + // picker/upload values that only carry storage fields. + if (id) return null + + const key = typeof record.key === 'string' ? record.key.trim() : '' + const path = typeof record.path === 'string' ? record.path.trim() : '' + const url = typeof record.url === 'string' ? record.url.trim() : '' + const fileUrl = + url || path || (key ? `/api/files/serve/${encodeURIComponent(key)}?context=workspace` : '') + + if (!fileUrl && !key) return null + + return { + id: key || fileUrl, + name: + typeof record.name === 'string' && record.name.trim() ? record.name.trim() : 'workspace-file', + url: fileUrl ? ensureAbsoluteUrl(fileUrl) : '', + size: typeof record.size === 'number' ? record.size : 0, + type: + typeof record.type === 'string' && record.type.trim() + ? record.type.trim() + : 'application/octet-stream', + key, + context: 'workspace', + } +} + +const normalizeFileIdList = (value: unknown): string[] => { + if (typeof value === 'string') { + const trimmed = value.trim() + if (!trimmed) return [] + + try { + return normalizeFileIdList(JSON.parse(trimmed)) + } catch { + return [trimmed] + } + } + + if (!Array.isArray(value)) return [] + + return value + .map((item) => (typeof item === 'string' ? item.trim() : '')) + .filter((id) => id.length > 0) +} + +const extractUserFilesFromInput = (fileInput: unknown) => { + const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : [] + return inputs + .map((input) => fileInputToUserFile(input)) + .filter((file): file is NonNullable> => Boolean(file)) +} + +const extractFileIdsFromInput = (fileInput: unknown): string[] => { + const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : [] + + return inputs + .flatMap((input) => { + if (typeof input === 'string') return normalizeFileIdList(input) + if (input && typeof input === 'object') { + const record = input as Record + if (typeof record.id === 'string') return normalizeFileIdList(record.id) + if (typeof record.fileId === 'string') return normalizeFileIdList(record.fileId) + } + return [] + }) + .filter((id) => id.length > 0) +} + export const POST = withRouteHandler(async (request: NextRequest) => { const auth = await checkInternalAuth(request, { requireWorkflowId: false }) if (!auth.success) { @@ -76,15 +170,52 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ success: true, data: { - file: { - id: file.id, - name: file.name, - url: ensureAbsoluteUrl(file.path), - size: file.size, - type: file.type, - key: file.key, - context: 'workspace', - }, + file: workspaceFileToUserFile(file), + }, + }) + } + + case 'read': { + const { fileId, fileInput } = body + const selectedFileIds = Array.isArray(fileId) + ? fileId.map((id) => id.trim()).filter(Boolean) + : fileId + ? normalizeFileIdList(fileId) + : extractFileIdsFromInput(fileInput) + const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput) + + if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) { + return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) + } + + const files = await Promise.all( + selectedFileIds.map((id) => getWorkspaceFile(workspaceId, id)) + ) + const missingFileId = selectedFileIds.find((_, index) => !files[index]) + if (missingFileId) { + return NextResponse.json( + { success: false, error: `File not found: "${missingFileId}"` }, + { status: 404 } + ) + } + + const userFiles = files + .map((file) => workspaceFileToUserFile(file)) + .filter((file): file is NonNullable> => + Boolean(file) + ) + .concat(selectedInputFiles) + + logger.info('Files retrieved', { + count: userFiles.length, + fileIds: userFiles.map((file) => file.id), + }) + + return NextResponse.json({ + success: true, + data: { + file: userFiles[0], + files: userFiles, }, }) } diff --git a/apps/sim/blocks/blocks.test.ts b/apps/sim/blocks/blocks.test.ts index 9659863a837..b39a6100783 100644 --- a/apps/sim/blocks/blocks.test.ts +++ b/apps/sim/blocks/blocks.test.ts @@ -94,6 +94,75 @@ describe.concurrent('Blocks Module', () => { }) }) + describe('File block', () => { + it('should keep v3 read and get routed to the legacy tools', () => { + const block = getBlock('file_v3') + + expect(block).toBeDefined() + expect(block?.hideFromToolbar).toBe(true) + expect(block?.subBlocks[0].options?.map((option) => option.id)).toEqual([ + 'file_parser_v3', + 'file_get', + 'file_write', + 'file_append', + ]) + expect(block?.tools.config?.tool({ operation: 'file_parser_v3' })).toBe('file_parser_v3') + expect(block?.tools.config?.tool({ operation: 'file_get' })).toBe('file_get') + }) + + it('should expose v4 with read and fetch routed to the expected tools', () => { + const block = getBlock('file_v4') + + expect(block).toBeDefined() + expect(block?.hideFromToolbar).toBe(false) + expect(block?.subBlocks[0].options?.map((option) => option.id)).toEqual([ + 'file_read', + 'file_fetch', + 'file_write', + 'file_append', + ]) + expect(block?.subBlocks.find((subBlock) => subBlock.id === 'readFile')?.multiple).toBe(true) + expect(block?.tools.config?.tool({ operation: 'file_read' })).toBe('file_read') + expect(block?.tools.config?.tool({ operation: 'file_fetch' })).toBe('file_fetch') + expect( + block?.tools.config?.params?.({ + operation: 'file_read', + readFileInput: '["file-1","file-2"]', + _context: { workspaceId: 'workspace-1' }, + }) + ).toEqual({ + fileId: ['file-1', 'file-2'], + workspaceId: 'workspace-1', + }) + expect( + block?.tools.config?.params?.({ + operation: 'file_read', + readFileInput: [ + { + key: 'workspace/workspace-1/example.md', + name: 'example.md', + path: '/api/files/serve/workspace%2Fworkspace-1%2Fexample.md?context=workspace', + size: 123, + type: 'text/markdown', + }, + ], + _context: { workspaceId: 'workspace-1' }, + }) + ).toEqual({ + fileInput: [ + { + key: 'workspace/workspace-1/example.md', + name: 'example.md', + path: '/api/files/serve/workspace%2Fworkspace-1%2Fexample.md?context=workspace', + size: 123, + type: 'text/markdown', + }, + ], + workspaceId: 'workspace-1', + }) + }) + }) + describe('getBlocksByCategory', () => { it('should return blocks in the "blocks" category', () => { const blocks = getBlocksByCategory('blocks') diff --git a/apps/sim/blocks/blocks/file.ts b/apps/sim/blocks/blocks/file.ts index efefb329824..43904e9816f 100644 --- a/apps/sim/blocks/blocks/file.ts +++ b/apps/sim/blocks/blocks/file.ts @@ -259,6 +259,7 @@ export const FileV3Block: BlockConfig = { tags: ['document-processing'], bgColor: '#40916C', icon: DocumentIcon, + hideFromToolbar: true, subBlocks: [ { id: 'operation', @@ -514,3 +515,291 @@ export const FileV3Block: BlockConfig = { }, }, } + +const parseReadFileIds = (input: unknown): string | string[] | null => { + let value = input + + if (typeof value === 'string') { + const trimmed = value.trim() + if (!trimmed) return null + + try { + value = JSON.parse(trimmed) + } catch { + return trimmed + } + } + + if (Array.isArray(value)) { + const fileIds = value + .map((item) => (typeof item === 'string' ? item.trim() : '')) + .filter((item) => item.length > 0) + + if (fileIds.length === 0) return null + return fileIds.length === 1 ? fileIds[0] : fileIds + } + + return null +} + +export const FileV4Block: BlockConfig = { + ...FileV3Block, + type: 'file_v4', + name: 'File', + description: 'Read, fetch, write, and append files', + longDescription: + 'Read workspace files by picker or canonical ID, fetch and parse files from URLs with optional headers, write new workspace files, or append content to existing files.', + hideFromToolbar: false, + bestPractices: ` + - Use Read when you need an existing workspace file object by picker selection or canonical file ID. + - Use Fetch for external file URLs. Add headers for authenticated downloads, for example Slack private file URLs require an Authorization Bearer token. + `, + subBlocks: [ + { + id: 'operation', + title: 'Operation', + type: 'dropdown' as SubBlockType, + options: [ + { label: 'Read', id: 'file_read' }, + { label: 'Fetch', id: 'file_fetch' }, + { label: 'Write', id: 'file_write' }, + { label: 'Append', id: 'file_append' }, + ], + value: () => 'file_read', + }, + { + id: 'readFile', + title: 'File', + type: 'file-upload' as SubBlockType, + canonicalParamId: 'readFileInput', + acceptedTypes: '*', + placeholder: 'Select workspace files', + multiple: true, + mode: 'basic', + condition: { field: 'operation', value: 'file_read' }, + required: { field: 'operation', value: 'file_read' }, + }, + { + id: 'readFileId', + title: 'File ID', + type: 'short-input' as SubBlockType, + canonicalParamId: 'readFileInput', + placeholder: 'Workspace file ID or JSON array of IDs', + mode: 'advanced', + condition: { field: 'operation', value: 'file_read' }, + required: { field: 'operation', value: 'file_read' }, + }, + { + id: 'fileUrl', + title: 'File URL', + type: 'short-input' as SubBlockType, + placeholder: 'https://example.com/document.pdf', + condition: { field: 'operation', value: 'file_fetch' }, + required: { field: 'operation', value: 'file_fetch' }, + }, + { + id: 'headers', + title: 'Headers', + type: 'table' as SubBlockType, + columns: ['Key', 'Value'], + description: + 'Custom headers for fetching the file URL, such as Authorization: Bearer .', + condition: { field: 'operation', value: 'file_fetch' }, + }, + { + id: 'fileName', + title: 'File Name', + type: 'short-input' as SubBlockType, + placeholder: 'File name (e.g., data.csv)', + condition: { field: 'operation', value: 'file_write' }, + required: { field: 'operation', value: 'file_write' }, + }, + { + id: 'content', + title: 'Content', + type: 'long-input' as SubBlockType, + placeholder: 'File content to write...', + condition: { field: 'operation', value: 'file_write' }, + required: { field: 'operation', value: 'file_write' }, + }, + { + id: 'contentType', + title: 'Content Type', + type: 'short-input' as SubBlockType, + placeholder: 'text/plain (auto-detected from extension)', + condition: { field: 'operation', value: 'file_write' }, + mode: 'advanced', + }, + { + id: 'appendFile', + title: 'File', + type: 'file-upload' as SubBlockType, + canonicalParamId: 'appendFileInput', + acceptedTypes: '.txt,.md,.json,.csv,.xml,.html,.htm,.yaml,.yml,.log,.rtf', + placeholder: 'Select or upload a workspace file', + mode: 'basic', + condition: { field: 'operation', value: 'file_append' }, + required: { field: 'operation', value: 'file_append' }, + }, + { + id: 'appendFileName', + title: 'File', + type: 'short-input' as SubBlockType, + canonicalParamId: 'appendFileInput', + placeholder: 'File name (e.g., notes.md)', + mode: 'advanced', + condition: { field: 'operation', value: 'file_append' }, + required: { field: 'operation', value: 'file_append' }, + }, + { + id: 'appendContent', + title: 'Content', + type: 'long-input' as SubBlockType, + placeholder: 'Content to append...', + condition: { field: 'operation', value: 'file_append' }, + required: { field: 'operation', value: 'file_append' }, + }, + ], + tools: { + access: ['file_fetch', 'file_read', 'file_write', 'file_append'], + config: { + tool: (params) => { + const operation = params.operation || 'file_read' + if (operation === 'file_read') return 'file_read' + if (operation === 'file_fetch') return 'file_fetch' + return operation + }, + params: (params) => { + const operation = params.operation || 'file_read' + + if (operation === 'file_write') { + return { + fileName: params.fileName, + content: params.content, + contentType: params.contentType, + workspaceId: params._context?.workspaceId, + } + } + + if (operation === 'file_append') { + const appendInput = params.appendFileInput + if (!appendInput) { + throw new Error('File is required for append') + } + + let fileName: string + if (typeof appendInput === 'string') { + fileName = appendInput.trim() + } else { + const normalized = normalizeFileInput(appendInput, { single: true }) + const file = normalized as Record | null + fileName = (file?.name as string) ?? '' + } + + if (!fileName) { + throw new Error('Could not determine file name') + } + + return { + fileName, + content: params.appendContent, + workspaceId: params._context?.workspaceId, + } + } + + if (operation === 'file_read') { + const readInput = params.readFileInput + if (!readInput) { + throw new Error('File is required for read') + } + + const fileIds = parseReadFileIds(readInput) + if (fileIds) { + return { + fileId: fileIds, + workspaceId: params._context?.workspaceId, + } + } + + const normalized = normalizeFileInput(readInput) + if (!normalized || normalized.length === 0) { + throw new Error('File is required for read') + } + + return { + fileInput: normalized, + workspaceId: params._context?.workspaceId, + } + } + + if (operation === 'file_fetch') { + const fileUrl = typeof params.fileUrl === 'string' ? params.fileUrl.trim() : '' + if (!fileUrl) { + logger.error('No file URL provided') + throw new Error('File URL is required') + } + + return { + filePath: fileUrl, + fileType: params.fileType || 'auto', + headers: params.headers, + workspaceId: params._context?.workspaceId, + workflowId: params._context?.workflowId, + executionId: params._context?.executionId, + } + } + + logger.error(`Invalid file operation: ${operation}`) + throw new Error('Invalid file operation') + }, + }, + }, + inputs: { + operation: { + type: 'string', + description: 'Operation to perform (read, fetch, write, or append)', + }, + readFileInput: { + type: 'json', + description: 'Selected workspace file or canonical file ID for read', + }, + fileUrl: { type: 'string', description: 'External file URL for fetch' }, + headers: { type: 'json', description: 'Request headers for fetch' }, + fileType: { type: 'string', description: 'File type for fetch' }, + fileName: { type: 'string', description: 'Name for a new file (write)' }, + content: { type: 'string', description: 'File content to write' }, + contentType: { type: 'string', description: 'MIME content type for write' }, + appendFileInput: { type: 'json', description: 'File to append to' }, + appendContent: { type: 'string', description: 'Content to append to file' }, + }, + outputs: { + file: { + type: 'file', + description: 'First workspace file object (read)', + }, + files: { + type: 'file[]', + description: 'Workspace file objects (read) or fetched file objects (fetch)', + }, + combinedContent: { + type: 'string', + description: 'All fetched file contents merged into a single text string (fetch)', + }, + id: { + type: 'string', + description: 'File ID (write)', + }, + name: { + type: 'string', + description: 'File name (write)', + }, + size: { + type: 'number', + description: 'File size in bytes (write)', + }, + url: { + type: 'string', + description: 'URL to access the file (write)', + }, + }, +} diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index d458289879a..cd6144ff5f7 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -55,7 +55,7 @@ import { EvernoteBlock } from '@/blocks/blocks/evernote' import { ExaBlock } from '@/blocks/blocks/exa' import { ExtendBlock, ExtendV2Block } from '@/blocks/blocks/extend' import { FathomBlock } from '@/blocks/blocks/fathom' -import { FileBlock, FileV2Block, FileV3Block } from '@/blocks/blocks/file' +import { FileBlock, FileV2Block, FileV3Block, FileV4Block } from '@/blocks/blocks/file' import { FirecrawlBlock } from '@/blocks/blocks/firecrawl' import { FirefliesBlock, FirefliesV2Block } from '@/blocks/blocks/fireflies' import { FunctionBlock } from '@/blocks/blocks/function' @@ -299,6 +299,7 @@ export const registry: Record = { file: FileBlock, file_v2: FileV2Block, file_v3: FileV3Block, + file_v4: FileV4Block, firecrawl: FirecrawlBlock, fireflies: FirefliesBlock, fireflies_v2: FirefliesV2Block, diff --git a/apps/sim/lib/api/contracts/storage-transfer.ts b/apps/sim/lib/api/contracts/storage-transfer.ts index 7cdc7edea56..7142c65ea90 100644 --- a/apps/sim/lib/api/contracts/storage-transfer.ts +++ b/apps/sim/lib/api/contracts/storage-transfer.ts @@ -302,6 +302,7 @@ export const fileParseBodySchema = z .object({ filePath: z.union([z.string(), z.array(z.string())]).optional(), fileType: z.string().optional().default(''), + headers: z.record(z.string(), z.string()).optional(), workspaceId: z.string().optional().default(''), workflowId: z.string().optional(), executionId: z.string().optional(), diff --git a/apps/sim/lib/api/contracts/tools/file.ts b/apps/sim/lib/api/contracts/tools/file.ts index 7120d38d15a..c91142a800a 100644 --- a/apps/sim/lib/api/contracts/tools/file.ts +++ b/apps/sim/lib/api/contracts/tools/file.ts @@ -33,10 +33,22 @@ export const fileManageGetBodySchema = z message: 'Either fileId or fileInput is required for get operation', }) +export const fileManageReadBodySchema = z + .object({ + operation: z.literal('read'), + workspaceId: z.string().min(1).optional(), + fileId: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]).optional(), + fileInput: z.any().optional(), + }) + .refine((data) => data.fileId !== undefined || data.fileInput !== undefined, { + message: 'Either fileId or fileInput is required for read operation', + }) + export const fileManageBodySchema = z.union([ fileManageWriteBodySchema, fileManageAppendBodySchema, fileManageGetBodySchema, + fileManageReadBodySchema, ]) export const fileManageContract = defineRouteContract({ diff --git a/apps/sim/tools/file/get.ts b/apps/sim/tools/file/get.ts index 6f05dbca7d2..3143e270343 100644 --- a/apps/sim/tools/file/get.ts +++ b/apps/sim/tools/file/get.ts @@ -7,6 +7,64 @@ interface FileGetParams { _context?: WorkflowToolExecutionContext } +interface FileReadParams { + fileId?: string | string[] + fileInput?: unknown + workspaceId?: string + _context?: WorkflowToolExecutionContext +} + +const createFileReadTool = (config: { + id: 'file_read' + name: string + description: string +}): ToolConfig => ({ + id: config.id, + name: config.name, + description: config.description, + version: '1.0.0', + + params: { + fileId: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Canonical workspace file ID, or an array of canonical workspace file IDs.', + }, + fileInput: { + type: 'file', + required: false, + visibility: 'user-only', + description: 'Selected workspace file object.', + }, + }, + + request: { + url: '/api/tools/file/manage', + method: 'POST', + headers: () => ({ 'Content-Type': 'application/json' }), + body: (params) => ({ + operation: 'read', + fileId: params.fileId, + fileInput: params.fileInput, + workspaceId: params.workspaceId || params._context?.workspaceId, + }), + }, + + transformResponse: async (response) => { + const data = await response.json() + if (!response.ok || !data.success) { + return { success: false, output: {}, error: data.error || 'Failed to get file' } + } + return { success: true, output: data.data } + }, + + outputs: { + file: { type: 'file', description: 'Workspace file object' }, + files: { type: 'file[]', description: 'Workspace file objects' }, + }, +}) + export const fileGetTool: ToolConfig = { id: 'file_get', name: 'File Get', @@ -52,3 +110,9 @@ export const fileGetTool: ToolConfig = { file: { type: 'file', description: 'Workspace file object' }, }, } + +export const fileReadTool = createFileReadTool({ + id: 'file_read', + name: 'File Read', + description: 'Read workspace file objects from selected files or canonical workspace file IDs.', +}) diff --git a/apps/sim/tools/file/index.ts b/apps/sim/tools/file/index.ts index 4e0b6daed0c..cde0e491b63 100644 --- a/apps/sim/tools/file/index.ts +++ b/apps/sim/tools/file/index.ts @@ -1,9 +1,15 @@ -import { fileParserTool, fileParserV2Tool, fileParserV3Tool } from '@/tools/file/parser' +import { + fileFetchTool, + fileParserTool, + fileParserV2Tool, + fileParserV3Tool, +} from '@/tools/file/parser' export { fileAppendTool } from '@/tools/file/append' -export { fileGetTool } from '@/tools/file/get' +export { fileGetTool, fileReadTool } from '@/tools/file/get' export { fileWriteTool } from '@/tools/file/write' export const fileParseTool = fileParserTool +export { fileFetchTool } export { fileParserV2Tool } export { fileParserV3Tool } diff --git a/apps/sim/tools/file/parser.ts b/apps/sim/tools/file/parser.ts index 5118e3f3a2d..d98e08de653 100644 --- a/apps/sim/tools/file/parser.ts +++ b/apps/sim/tools/file/parser.ts @@ -10,6 +10,7 @@ import type { FileParserV3OutputData, FileUploadInput, } from '@/tools/file/types' +import { transformTable } from '@/tools/shared/table' import type { ToolConfig } from '@/tools/types' const logger = createLogger('FileParserTool') @@ -34,6 +35,20 @@ const isFileParseResult = (value: unknown): value is FileParseResult => typeof value.name === 'string' && typeof value.binary === 'boolean' +const normalizeHeaders = (headers: FileParserInput['headers']): Record => { + const transformed = transformTable(headers ?? null) + return Object.entries(transformed).reduce( + (acc, [key, value]) => { + const headerName = key.trim() + if (headerName && value !== undefined && value !== null) { + acc[headerName] = String(value) + } + return acc + }, + {} as Record + ) +} + const normalizeFileParseResult = (value: unknown): FileParseResult => { if (isRecord(value) && isFileParseResult(value.output)) { return value.output @@ -245,9 +260,11 @@ export const fileParserTool: ToolConfig = { } logger.info('Tool body determined filePath:', determinedFilePath) + const headers = normalizeHeaders(params.headers) return { filePath: determinedFilePath, fileType: determinedFileType, + ...(Object.keys(headers).length > 0 && { headers }), workspaceId: params.workspaceId || params._context?.workspaceId, workflowId: params._context?.workflowId, executionId: params._context?.executionId, @@ -286,6 +303,25 @@ export const fileParserV2Tool: ToolConfig = { }, } +const parseFileParserV3Response = async (response: Response): Promise => { + const parsed = await parseFileParserResponse(response) + const output = parsed.output as FileParserOutputData + const files = + Array.isArray(output.processedFiles) && output.processedFiles.length > 0 + ? output.processedFiles + : [] + + const cleanedOutput: FileParserV3OutputData = { + files, + combinedContent: output.combinedContent, + } + + return { + success: true, + output: cleanedOutput, + } +} + export const fileParserV3Tool: ToolConfig = { id: 'file_parser_v3', name: 'File Parser', @@ -293,26 +329,31 @@ export const fileParserV3Tool: ToolConfig = version: '3.0.0', params: fileParserTool.params, request: fileParserTool.request, - transformResponse: async (response: Response): Promise => { - const parsed = await parseFileParserResponse(response) - const output = parsed.output as FileParserOutputData - const files = - Array.isArray(output.processedFiles) && output.processedFiles.length > 0 - ? output.processedFiles - : [] - - const cleanedOutput: FileParserV3OutputData = { - files, - combinedContent: output.combinedContent, - } - - return { - success: true, - output: cleanedOutput, - } - }, + transformResponse: parseFileParserV3Response, outputs: { files: { type: 'file[]', description: 'Parsed files as UserFile objects' }, combinedContent: { type: 'string', description: 'Combined content of all parsed files' }, }, } + +export const fileFetchTool: ToolConfig = { + id: 'file_fetch', + name: 'File Fetch', + description: 'Fetch and parse a file from a URL with optional custom headers.', + version: '1.0.0', + params: { + ...fileParserTool.params, + headers: { + type: 'object', + required: false, + visibility: 'user-or-llm', + description: 'HTTP headers to include when fetching URL-based files.', + }, + }, + request: fileParserTool.request, + transformResponse: parseFileParserV3Response, + outputs: { + files: { type: 'file[]', description: 'Fetched files as UserFile objects' }, + combinedContent: { type: 'string', description: 'Combined content of all fetched files' }, + }, +} diff --git a/apps/sim/tools/file/types.ts b/apps/sim/tools/file/types.ts index 73553c6e02f..7943c470444 100644 --- a/apps/sim/tools/file/types.ts +++ b/apps/sim/tools/file/types.ts @@ -1,10 +1,11 @@ import type { UserFile } from '@/executor/types' -import type { ToolResponse } from '@/tools/types' +import type { TableRow, ToolResponse } from '@/tools/types' export interface FileParserInput { filePath?: string | string[] file?: UserFile | UserFile[] | FileUploadInput | FileUploadInput[] fileType?: string + headers?: TableRow[] | Record | string | null workspaceId?: string workflowId?: string executionId?: string diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index c71385aa0d9..52e6db1b6bc 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -645,10 +645,12 @@ import { } from '@/tools/fathom' import { fileAppendTool, + fileFetchTool, fileGetTool, fileParserV2Tool, fileParserV3Tool, fileParseTool, + fileReadTool, fileWriteTool, } from '@/tools/file' import { @@ -3216,7 +3218,9 @@ export const tools: Record = { file_parser_v2: fileParserV2Tool, file_parser_v3: fileParserV3Tool, file_append: fileAppendTool, + file_fetch: fileFetchTool, file_get: fileGetTool, + file_read: fileReadTool, file_write: fileWriteTool, firecrawl_scrape: firecrawlScrapeTool, firecrawl_search: firecrawlSearchTool, From f74dadedfd8e2ac96c50dbf6d31939913d421f07 Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan Date: Thu, 14 May 2026 18:17:40 -0700 Subject: [PATCH 2/6] Support files in agent block --- apps/sim/blocks/blocks.test.ts | 36 ++ apps/sim/blocks/blocks/agent.ts | 28 +- apps/sim/executor/execution/engine.ts | 13 - .../handlers/agent/agent-handler.test.ts | 72 +++ .../executor/handlers/agent/agent-handler.ts | 68 +- apps/sim/executor/handlers/agent/types.ts | 4 + apps/sim/lib/uploads/utils/file-utils.ts | 4 +- apps/sim/providers/anthropic/core.ts | 4 +- apps/sim/providers/attachments.test.ts | 271 ++++++++ apps/sim/providers/attachments.ts | 604 ++++++++++++++++++ apps/sim/providers/azure-openai/index.ts | 3 + apps/sim/providers/bedrock/index.ts | 7 +- apps/sim/providers/cerebras/index.ts | 6 +- apps/sim/providers/deepseek/index.ts | 6 +- apps/sim/providers/fireworks/index.ts | 6 +- apps/sim/providers/gemini/core.ts | 2 +- apps/sim/providers/google/utils.test.ts | 38 ++ apps/sim/providers/google/utils.ts | 11 +- apps/sim/providers/groq/index.ts | 6 +- apps/sim/providers/mistral/index.ts | 6 +- apps/sim/providers/ollama/index.ts | 6 +- apps/sim/providers/openai/core.ts | 2 +- apps/sim/providers/openai/utils.test.ts | 40 ++ apps/sim/providers/openai/utils.ts | 26 +- apps/sim/providers/openrouter/index.ts | 6 +- apps/sim/providers/types.ts | 3 +- apps/sim/providers/vllm/index.ts | 6 +- apps/sim/providers/xai/index.ts | 6 +- 28 files changed, 1239 insertions(+), 51 deletions(-) create mode 100644 apps/sim/providers/attachments.test.ts create mode 100644 apps/sim/providers/attachments.ts create mode 100644 apps/sim/providers/openai/utils.test.ts diff --git a/apps/sim/blocks/blocks.test.ts b/apps/sim/blocks/blocks.test.ts index b39a6100783..8906f962ef3 100644 --- a/apps/sim/blocks/blocks.test.ts +++ b/apps/sim/blocks/blocks.test.ts @@ -163,6 +163,42 @@ describe.concurrent('Blocks Module', () => { }) }) + describe('Agent block', () => { + it('should expose canonical file attachments and normalize file params', () => { + const block = getBlock('agent') + + expect(block).toBeDefined() + const uploadSubBlock = block?.subBlocks.find((subBlock) => subBlock.id === 'attachmentFiles') + const advancedSubBlock = block?.subBlocks.find((subBlock) => subBlock.id === 'files') + + expect(uploadSubBlock?.type).toBe('file-upload') + expect(uploadSubBlock?.canonicalParamId).toBe('files') + expect(uploadSubBlock?.multiple).toBe(true) + expect(advancedSubBlock?.canonicalParamId).toBe('files') + expect(block?.inputs.files).toEqual({ + type: 'array', + description: 'Files to include with the latest user message', + }) + + expect( + block?.tools.config?.params?.({ + model: 'gpt-4o', + files: + '[{"id":"file-1","key":"workspace/ws-1/example.png","name":"example.png","url":"/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace","size":123,"type":"image/png"}]', + }) + ).toMatchObject({ + files: [ + { + id: 'file-1', + key: 'workspace/ws-1/example.png', + name: 'example.png', + type: 'image/png', + }, + ], + }) + }) + }) + describe('getBlocksByCategory', () => { it('should return blocks in the "blocks" category', () => { const blocks = getBlocksByCategory('blocks') diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index 3a8e704859d..652270de636 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -5,6 +5,7 @@ import { AuthMode, IntegrationType } from '@/blocks/types' import { getModelOptions, getProviderCredentialSubBlocks, + normalizeFileInput, RESPONSE_FORMAT_WAND_CONFIG, } from '@/blocks/utils' import { @@ -133,6 +134,25 @@ Return ONLY the JSON array.`, defaultValue: 'claude-sonnet-4-6', options: getModelOptions, }, + { + id: 'attachmentFiles', + title: 'Files', + type: 'file-upload', + canonicalParamId: 'files', + placeholder: 'Upload files for the agent', + multiple: true, + mode: 'basic', + required: false, + }, + { + id: 'files', + title: 'Files', + type: 'short-input', + canonicalParamId: 'files', + placeholder: 'Reference files from previous blocks', + mode: 'advanced', + required: false, + }, { id: 'reasoningEffort', title: 'Reasoning Effort', @@ -472,6 +492,9 @@ Return ONLY the JSON array.`, return tool }, params: (params: Record) => { + const normalizedFiles = normalizeFileInput(params.files) + const baseParams = normalizedFiles ? { ...params, files: normalizedFiles } : params + // If tools array is provided, handle tool usage control if (params.tools && Array.isArray(params.tools)) { // Transform tools to include usageControl @@ -506,9 +529,9 @@ Return ONLY the JSON array.`, logger.info('Filtered out tools set to none', { tools: filteredOutTools.join(', ') }) } - return { ...params, tools: transformedTools } + return { ...baseParams, tools: transformedTools } } - return params + return baseParams }, }, }, @@ -518,6 +541,7 @@ Return ONLY the JSON array.`, description: 'Array of message objects with role and content: [{ role: "system", content: "..." }, { role: "user", content: "..." }]', }, + files: { type: 'array', description: 'Files to include with the latest user message' }, memoryType: { type: 'string', description: diff --git a/apps/sim/executor/execution/engine.ts b/apps/sim/executor/execution/engine.ts index 82497858911..45880bc44d2 100644 --- a/apps/sim/executor/execution/engine.ts +++ b/apps/sim/executor/execution/engine.ts @@ -449,19 +449,6 @@ export class ExecutionEngine { const readyNodes = this.edgeManager.processOutgoingEdges(node, output, false) - this.execLogger.info('Processing outgoing edges', { - nodeId, - outgoingEdgesCount: node.outgoingEdges.size, - outgoingEdges: Array.from(node.outgoingEdges.entries()).map(([id, e]) => ({ - id, - target: e.target, - sourceHandle: e.sourceHandle, - })), - output, - readyNodesCount: readyNodes.length, - readyNodes, - }) - this.addMultipleToQueue(readyNodes) if (this.context.pendingDynamicNodes && this.context.pendingDynamicNodes.length > 0) { diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index 169cb4f52f9..2d7a89c0a87 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -264,6 +264,78 @@ describe('AgentBlockHandler', () => { expect(result).toEqual(expectedOutput) }) + it('should attach files to the last user message only', async () => { + const inputs = { + model: 'gpt-4o', + messages: [ + { role: 'system' as const, content: 'You are helpful.' }, + { role: 'user' as const, content: 'Earlier question' }, + { role: 'assistant' as const, content: 'Earlier answer' }, + { role: 'user' as const, content: 'Analyze this file' }, + ], + files: [ + { + id: 'file-1', + key: 'workspace/ws-1/example.png', + name: 'example.png', + url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace', + size: 128, + type: 'image/png', + base64: 'aW1hZ2U=', + }, + ], + apiKey: 'test-api-key', + } + + mockGetProviderFromModel.mockReturnValue('openai') + + await handler.execute(mockContext, mockBlock, inputs) + + const requestBody = mockExecuteProviderRequest.mock.calls[0][1] + expect(requestBody.messages[1]).toMatchObject({ + role: 'user', + content: 'Earlier question', + }) + expect(requestBody.messages[1].files).toBeUndefined() + expect(requestBody.messages[3]).toMatchObject({ + role: 'user', + content: 'Analyze this file', + files: [ + { + id: 'file-1', + name: 'example.png', + type: 'image/png', + base64: 'aW1hZ2U=', + }, + ], + }) + }) + + it('should reject files for providers without attachment support', async () => { + const inputs = { + model: 'deepseek-chat', + messages: [{ role: 'user' as const, content: 'Analyze this file' }], + files: [ + { + id: 'file-1', + key: 'workspace/ws-1/example.png', + name: 'example.png', + url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace', + size: 128, + type: 'image/png', + base64: 'aW1hZ2U=', + }, + ], + apiKey: 'test-api-key', + } + + mockGetProviderFromModel.mockReturnValue('deepseek') + + await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( + 'File attachments are not supported for provider "deepseek"' + ) + }) + it('should preserve usageControl for custom tools and filter out "none"', async () => { const inputs = { model: 'gpt-4o', diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index e09b75387f7..08a95be5195 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -6,9 +6,12 @@ import { sleep } from '@sim/utils/helpers' import { and, eq, inArray, isNull } from 'drizzle-orm' import { normalizeStringRecord, normalizeWorkflowVariables } from '@/lib/core/utils/records' import { createMcpToolId } from '@/lib/mcp/utils' +import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' +import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' import { getCustomToolById } from '@/lib/workflows/custom-tools/operations' import { getAllBlocks } from '@/blocks' import type { BlockOutput } from '@/blocks/types' +import { normalizeFileInput } from '@/blocks/utils' import { validateBlockType, validateCustomToolsAllowed, @@ -36,6 +39,7 @@ import { buildAPIUrl, buildAuthHeaders } from '@/executor/utils/http' import { stringifyJSON } from '@/executor/utils/json' import { resolveVertexCredential } from '@/executor/utils/vertex-credential' import { executeProviderRequest } from '@/providers' +import { getAttachmentProvider, getProviderAttachmentMaxBytes } from '@/providers/attachments' import { getProviderFromModel, transformBlockTool } from '@/providers/utils' import type { SerializedBlock } from '@/serializer/types' import { filterSchemaForLLM, type ToolSchema } from '@/tools/params' @@ -87,12 +91,18 @@ export class AgentBlockHandler implements BlockHandler { const streamingConfig = this.getStreamingConfig(ctx, block) const messages = await this.buildMessages(ctx, filteredInputs, skillMetadata) + const messagesWithFiles = await this.attachFilesToLastUserMessage( + ctx, + messages, + filteredInputs.files, + providerId + ) const providerRequest = this.buildProviderRequest({ ctx, providerId, model, - messages, + messages: messagesWithFiles, inputs: filteredInputs, formattedTools, responseFormat, @@ -672,6 +682,62 @@ export class AgentBlockHandler implements BlockHandler { return messages.length > 0 ? messages : undefined } + private async attachFilesToLastUserMessage( + ctx: ExecutionContext, + messages: Message[] | undefined, + filesInput: unknown, + providerId: string + ): Promise { + const normalizedFiles = normalizeFileInput(filesInput) + if (!normalizedFiles || normalizedFiles.length === 0) { + return messages + } + + if (!messages || messages.length === 0) { + throw new Error('Files require at least one user message in the agent prompt') + } + + if (!getAttachmentProvider(providerId)) { + throw new Error(`File attachments are not supported for provider "${providerId}"`) + } + + let lastUserMessageIndex = -1 + for (let index = messages.length - 1; index >= 0; index--) { + if (messages[index].role === 'user') { + lastUserMessageIndex = index + break + } + } + if (lastUserMessageIndex === -1) { + throw new Error('Files require at least one user message in the agent prompt') + } + + const requestId = ctx.executionId || ctx.workflowId || 'agent-files' + const userFiles = processFilesToUserFiles(normalizedFiles as RawFileInput[], requestId, logger) + if (userFiles.length === 0) { + throw new Error('Files must include at least one valid file object') + } + + const hydratedFiles = await hydrateUserFilesWithBase64(userFiles, { + requestId, + workspaceId: ctx.workspaceId, + workflowId: ctx.workflowId, + executionId: ctx.executionId, + userId: ctx.userId, + logger, + maxBytes: getProviderAttachmentMaxBytes(providerId), + }) + + const lastUserMessage = messages[lastUserMessageIndex] + const nextMessages = [...messages] + nextMessages[lastUserMessageIndex] = { + ...lastUserMessage, + files: [...(lastUserMessage.files ?? []), ...hydratedFiles], + } + + return nextMessages + } + private extractValidMessages(messages?: Message[]): Message[] { if (!messages || !Array.isArray(messages)) return [] diff --git a/apps/sim/executor/handlers/agent/types.ts b/apps/sim/executor/handlers/agent/types.ts index 7f4a185bacc..39329739a43 100644 --- a/apps/sim/executor/handlers/agent/types.ts +++ b/apps/sim/executor/handlers/agent/types.ts @@ -1,3 +1,5 @@ +import type { UserFile } from '@/executor/types' + export interface SkillInput { skillId: string name?: string @@ -37,6 +39,7 @@ export interface AgentInputs { reasoningEffort?: string verbosity?: string thinkingLevel?: string + files?: unknown } /** @@ -66,6 +69,7 @@ export interface ToolInput { export interface Message { role: 'system' | 'user' | 'assistant' content: string + files?: UserFile[] executionId?: string function_call?: any tool_calls?: any[] diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index cf49299f5d5..cad992fa94b 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -172,7 +172,7 @@ export function createFileContentFromBase64(base64: string, mimeType: string): M return null } - if (contentType === 'image' && !CLAUDE_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType.toLowerCase())) { + if (contentType === 'image' && !MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType.toLowerCase())) { return null } @@ -186,7 +186,7 @@ export function createFileContentFromBase64(base64: string, mimeType: string): M } } -const CLAUDE_SUPPORTED_IMAGE_MIME_TYPES = new Set([ +export const MODEL_SUPPORTED_IMAGE_MIME_TYPES = new Set([ 'image/jpeg', 'image/jpg', 'image/png', diff --git a/apps/sim/providers/anthropic/core.ts b/apps/sim/providers/anthropic/core.ts index bda5c2f6f4a..5d5951d8ae4 100644 --- a/apps/sim/providers/anthropic/core.ts +++ b/apps/sim/providers/anthropic/core.ts @@ -9,6 +9,7 @@ import { checkForForcedToolUsage, createReadableStreamFromAnthropicStream, } from '@/providers/anthropic/utils' +import { buildAnthropicMessageContent } from '@/providers/attachments' import { getMaxOutputTokensForModel, getThinkingCapability, @@ -229,9 +230,10 @@ export async function executeAnthropicProviderRequest( ], }) } else { + const content = buildAnthropicMessageContent(msg.content, msg.files, config.providerId) messages.push({ role: msg.role === 'assistant' ? 'assistant' : 'user', - content: msg.content ? [{ type: 'text', text: msg.content }] : [], + content: content as unknown as Anthropic.Messages.ContentBlockParam[], }) } }) diff --git a/apps/sim/providers/attachments.test.ts b/apps/sim/providers/attachments.test.ts new file mode 100644 index 00000000000..813a7c54b53 --- /dev/null +++ b/apps/sim/providers/attachments.test.ts @@ -0,0 +1,271 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import type { UserFile } from '@/executor/types' +import { + buildAnthropicMessageContent, + buildBedrockMessageContent, + buildGeminiMessageParts, + buildOpenAIMessageContent, + buildOpenRouterMessageContent, + formatMessagesForProvider, + inferAttachmentMimeType, + prepareProviderAttachments, +} from '@/providers/attachments' + +const imageFile: UserFile = { + id: 'file-1', + name: 'example.png', + url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace', + size: 128, + type: 'image/png', + key: 'workspace/ws-1/example.png', + base64: 'iVBORw0KGgo=', +} + +const pdfFile: UserFile = { + id: 'file-2', + name: 'example.pdf', + url: '/api/files/serve/workspace%2Fws-1%2Fexample.pdf?context=workspace', + size: 256, + type: 'application/pdf', + key: 'workspace/ws-1/example.pdf', + base64: 'cGRm', +} + +const markdownFile: UserFile = { + id: 'file-3', + name: 'notes.md', + url: '/api/files/serve/workspace%2Fws-1%2Fnotes.md?context=workspace', + size: 17, + type: 'text/markdown', + key: 'workspace/ws-1/notes.md', + base64: Buffer.from('# Notes\n\nHello').toString('base64'), +} + +describe('provider attachments', () => { + it('infers MIME type from filename when file type is generic', () => { + expect( + inferAttachmentMimeType({ + ...imageFile, + type: 'application/octet-stream', + }) + ).toBe('image/png') + }) + + it('formats OpenAI Responses content with text, image, and file parts', () => { + const content = buildOpenAIMessageContent( + 'Analyze these files', + [imageFile, pdfFile, markdownFile], + 'openai' + ) + + expect(content).toEqual([ + { type: 'input_text', text: 'Analyze these files' }, + { + type: 'input_image', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + }, + { + type: 'input_file', + filename: 'example.pdf', + file_data: 'data:application/pdf;base64,cGRm', + }, + { + type: 'input_file', + filename: 'notes.md', + file_data: `data:text/markdown;base64,${markdownFile.base64}`, + }, + ]) + }) + + it('formats Anthropic content with image, PDF document, and text document blocks', () => { + const content = buildAnthropicMessageContent( + 'Analyze these files', + [imageFile, pdfFile, markdownFile], + 'anthropic' + ) + + expect(content).toEqual([ + { type: 'text', text: 'Analyze these files' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgo=', + }, + }, + { + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'cGRm', + }, + title: 'example.pdf', + }, + { + type: 'document', + source: { + type: 'text', + media_type: 'text/plain', + data: '# Notes\n\nHello', + }, + title: 'notes.md', + }, + ]) + }) + + it('formats Gemini content with text and inline data parts', () => { + const parts = buildGeminiMessageParts('Analyze this file', [imageFile, markdownFile], 'google') + + expect(parts).toEqual([ + { text: 'Analyze this file' }, + { + inlineData: { + mimeType: 'image/png', + data: 'iVBORw0KGgo=', + }, + }, + { + inlineData: { + mimeType: 'text/plain', + data: markdownFile.base64, + }, + }, + ]) + }) + + it('formats Bedrock content with native document blocks', () => { + const parts = buildBedrockMessageContent('Analyze this file', [markdownFile], 'bedrock') + + expect(parts).toEqual([ + { text: 'Analyze this file' }, + { + document: { + format: 'md', + name: 'notes', + source: { + bytes: Buffer.from(markdownFile.base64, 'base64'), + }, + }, + }, + ]) + }) + + it('formats OpenRouter images and PDFs with native multimodal message parts', () => { + const content = buildOpenRouterMessageContent( + 'Analyze these files', + [imageFile, pdfFile], + 'openrouter' + ) + + expect(content).toEqual([ + { type: 'text', text: 'Analyze these files' }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + }, + { + type: 'file', + file: { + filename: 'example.pdf', + file_data: 'data:application/pdf;base64,cGRm', + }, + }, + ]) + }) + + it('formats image-only provider messages and strips file fields', () => { + const messages = formatMessagesForProvider( + [{ role: 'user', content: 'Analyze this image', files: [imageFile] }], + 'groq' + ) + + expect(messages).toEqual([ + { + role: 'user', + content: [ + { type: 'text', text: 'Analyze this image' }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + }, + ], + }, + ]) + }) + + it('fails fast for unsupported MIME types', () => { + expect(() => + prepareProviderAttachments( + [ + { + ...imageFile, + name: 'archive.zip', + type: 'application/zip', + }, + ], + 'openai' + ) + ).toThrow('application/zip') + }) + + it('sniffs image bytes and corrects a wrong declared image MIME type', () => { + const content = buildAnthropicMessageContent( + 'Analyze this image', + [ + { + ...imageFile, + name: 'wrong.ico', + type: 'image/x-icon', + }, + ], + 'anthropic' + ) + + expect(content[1]).toEqual({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgo=', + }, + }) + }) + + it('rejects image attachments when the bytes are not a supported image format', () => { + expect(() => + prepareProviderAttachments( + [ + { + ...imageFile, + name: 'not-an-image.png', + base64: Buffer.from('not an image').toString('base64'), + }, + ], + 'anthropic' + ) + ).toThrow('not a supported model image format') + }) + + it('rejects documents for image-only providers', () => { + expect(() => + formatMessagesForProvider( + [{ role: 'user', content: 'Analyze this file', files: [pdfFile] }], + 'groq' + ) + ).toThrow('Supported attachments: images') + }) + + it('rejects providers without file attachment support', () => { + expect(() => + formatMessagesForProvider( + [{ role: 'user', content: 'Analyze this file', files: [imageFile] }], + 'deepseek' + ) + ).toThrow('not supported') + }) +}) diff --git a/apps/sim/providers/attachments.ts b/apps/sim/providers/attachments.ts new file mode 100644 index 00000000000..eda2b552b48 --- /dev/null +++ b/apps/sim/providers/attachments.ts @@ -0,0 +1,604 @@ +import { + getContentType, + getFileExtension, + getMimeTypeFromExtension, + MODEL_SUPPORTED_IMAGE_MIME_TYPES, +} from '@/lib/uploads/utils/file-utils' +import type { UserFile } from '@/executor/types' +import type { ProviderId } from '@/providers/types' + +export type AttachmentProvider = + | 'openai' + | 'anthropic' + | 'google' + | 'bedrock' + | 'openrouter' + | 'mistral' + | 'groq' + | 'fireworks' + | 'ollama' + | 'vllm' + | 'xai' + | 'deepseek' + | 'cerebras' + +export interface PreparedProviderAttachment { + file: UserFile + filename: string + mimeType: string + providerMimeType: string + base64: string + dataUrl: string + text?: string + extension: string + contentType: 'image' | 'document' | 'audio' | 'video' +} + +const AGENT_ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024 +const PDF_MIME_TYPE = 'application/pdf' + +const TEXT_DOCUMENT_MIME_TYPES = new Set([ + 'text/plain', + 'text/markdown', + 'text/csv', + 'text/html', + 'text/xml', + 'text/javascript', + 'text/typescript', + 'text/x-python', + 'text/x-go', + 'text/x-rust', + 'text/x-java', + 'text/x-kotlin', + 'text/x-c', + 'text/x-c++', + 'text/x-csharp', + 'text/x-ruby', + 'text/x-php', + 'text/x-swift', + 'text/x-shellscript', + 'application/json', + 'application/xml', + 'application/x-yaml', +]) + +const OPENAI_DOCUMENT_MIME_TYPES = new Set([ + PDF_MIME_TYPE, + ...TEXT_DOCUMENT_MIME_TYPES, + 'application/rtf', + 'application/msword', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.ms-powerpoint', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'application/vnd.ms-excel', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', +]) + +const GEMINI_INLINE_MIME_TYPES = new Set([ + PDF_MIME_TYPE, + ...TEXT_DOCUMENT_MIME_TYPES, + ...MODEL_SUPPORTED_IMAGE_MIME_TYPES, + 'audio/mpeg', + 'audio/mp3', + 'audio/mp4', + 'audio/x-m4a', + 'audio/m4a', + 'audio/wav', + 'audio/wave', + 'audio/x-wav', + 'audio/webm', + 'audio/ogg', + 'audio/flac', + 'video/mp4', + 'video/mpeg', + 'video/quicktime', + 'video/x-quicktime', + 'video/webm', +]) + +const BEDROCK_DOCUMENT_FORMATS = new Set([ + 'pdf', + 'csv', + 'doc', + 'docx', + 'xls', + 'xlsx', + 'html', + 'txt', + 'md', +]) +const BEDROCK_IMAGE_FORMATS = new Set(['png', 'jpeg', 'jpg', 'gif', 'webp']) +const BEDROCK_VIDEO_FORMATS = new Set(['mp4', 'mov', 'mkv', 'webm']) + +const IMAGE_ONLY_PROVIDERS = new Set([ + 'mistral', + 'groq', + 'fireworks', + 'ollama', + 'vllm', +]) + +const UNSUPPORTED_FILE_PROVIDERS = new Set(['xai', 'deepseek', 'cerebras']) + +const PROVIDER_SUPPORTED_LABELS: Record = { + openai: 'images and documents through the Responses API input_image/input_file parts', + anthropic: 'images, PDFs, and text documents through Claude content blocks', + google: 'images, audio, video, PDFs, and text documents through Gemini inlineData', + bedrock: 'Bedrock Converse image, document, and video content blocks', + openrouter: 'images and PDFs through OpenRouter multimodal message parts', + mistral: 'images through image_url message parts', + groq: 'images through image_url message parts on multimodal models', + fireworks: 'images through image_url message parts on vision models', + ollama: 'images through image_url message parts on vision models', + vllm: 'images through image_url message parts on multimodal models', + xai: 'no file attachments in the current chat-completions adapter', + deepseek: 'no file attachments in the current API adapter', + cerebras: 'no file attachments in the current API adapter', +} + +export function getAttachmentProvider(providerId: ProviderId | string): AttachmentProvider | null { + if (providerId === 'openai' || providerId === 'azure-openai') return 'openai' + if (providerId === 'anthropic' || providerId === 'azure-anthropic') return 'anthropic' + if (providerId === 'google' || providerId === 'vertex') return 'google' + if (providerId === 'bedrock') return 'bedrock' + if (providerId === 'openrouter') return 'openrouter' + if (providerId === 'mistral') return 'mistral' + if (providerId === 'groq') return 'groq' + if (providerId === 'fireworks') return 'fireworks' + if (providerId === 'ollama') return 'ollama' + if (providerId === 'vllm') return 'vllm' + if (providerId === 'xai') return 'xai' + if (providerId === 'deepseek') return 'deepseek' + if (providerId === 'cerebras') return 'cerebras' + return null +} + +export function getProviderAttachmentMaxBytes(_providerId: ProviderId | string): number { + return AGENT_ATTACHMENT_MAX_BYTES +} + +export function inferAttachmentMimeType(file: UserFile): string { + const explicitType = file.type?.trim().toLowerCase() + if (explicitType && explicitType !== 'application/octet-stream') { + return explicitType + } + + const inferred = getMimeTypeFromExtension(getFileExtension(file.name)) + return inferred.toLowerCase() +} + +function isTextDocumentMimeType(mimeType: string): boolean { + return TEXT_DOCUMENT_MIME_TYPES.has(mimeType) || mimeType.startsWith('text/') +} + +function isImageMimeType(mimeType: string): boolean { + return MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType) +} + +function isOpenAIDocumentMimeType(mimeType: string): boolean { + return OPENAI_DOCUMENT_MIME_TYPES.has(mimeType) || isTextDocumentMimeType(mimeType) +} + +function sniffImageMimeType(base64: string): string { + let bytes: Buffer + try { + bytes = Buffer.from(base64, 'base64') + } catch { + return '' + } + + if ( + bytes.length >= 8 && + bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) + ) { + return 'image/png' + } + + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return 'image/jpeg' + } + + if ( + bytes.length >= 6 && + (bytes.subarray(0, 6).equals(Buffer.from('GIF87a')) || + bytes.subarray(0, 6).equals(Buffer.from('GIF89a'))) + ) { + return 'image/gif' + } + + if ( + bytes.length >= 12 && + bytes.subarray(0, 4).equals(Buffer.from('RIFF')) && + bytes.subarray(8, 12).equals(Buffer.from('WEBP')) + ) { + return 'image/webp' + } + + return '' +} + +function getAttachmentExtension(file: UserFile, mimeType: string): string { + if (mimeType === 'image/jpeg' || mimeType === 'image/jpg') return 'jpeg' + if (mimeType === 'image/png') return 'png' + if (mimeType === 'image/gif') return 'gif' + if (mimeType === 'image/webp') return 'webp' + if (mimeType === 'video/mp4') return 'mp4' + if (mimeType === 'video/quicktime' || mimeType === 'video/x-quicktime') return 'mov' + if (mimeType === 'video/webm') return 'webm' + + const extension = getFileExtension(file.name) + if (extension) return extension + + if (mimeType === 'application/pdf') return 'pdf' + if (mimeType === 'text/markdown') return 'md' + if (mimeType === 'text/plain') return 'txt' + if (mimeType === 'text/csv') return 'csv' + if (mimeType === 'text/html') return 'html' + return '' +} + +function normalizeProviderMimeType(mimeType: string, provider: AttachmentProvider): string { + if ((provider === 'anthropic' || provider === 'google') && isTextDocumentMimeType(mimeType)) { + return 'text/plain' + } + return mimeType +} + +function decodeBase64Text(base64: string, filename: string): string { + try { + return Buffer.from(base64, 'base64').toString('utf8') + } catch { + throw new Error(`File "${filename}" could not be decoded as UTF-8 text`) + } +} + +function toDataUrl(mimeType: string, base64: string): string { + return `data:${mimeType};base64,${base64}` +} + +function getProviderSupportedLabel(provider: AttachmentProvider): string { + return PROVIDER_SUPPORTED_LABELS[provider] +} + +function validateProviderSupport( + attachment: Omit, + provider: AttachmentProvider, + providerId: ProviderId | string +) { + const { filename, mimeType, contentType, extension } = attachment + const supportedLabel = getProviderSupportedLabel(provider) + + const supported = + provider === 'openai' + ? isImageMimeType(mimeType) || isOpenAIDocumentMimeType(mimeType) + : provider === 'anthropic' + ? isImageMimeType(mimeType) || + mimeType === PDF_MIME_TYPE || + isTextDocumentMimeType(mimeType) + : provider === 'google' + ? GEMINI_INLINE_MIME_TYPES.has(mimeType) || isTextDocumentMimeType(mimeType) + : provider === 'bedrock' + ? (contentType === 'image' && BEDROCK_IMAGE_FORMATS.has(extension)) || + (contentType === 'document' && BEDROCK_DOCUMENT_FORMATS.has(extension)) || + (contentType === 'video' && BEDROCK_VIDEO_FORMATS.has(extension)) + : provider === 'openrouter' + ? isImageMimeType(mimeType) || mimeType === PDF_MIME_TYPE + : IMAGE_ONLY_PROVIDERS.has(provider) + ? isImageMimeType(mimeType) + : !UNSUPPORTED_FILE_PROVIDERS.has(provider) + + if (!supported) { + throw new Error( + `File "${filename}" has MIME type "${mimeType}", which is not supported by provider "${providerId}". Supported attachments: ${supportedLabel}.` + ) + } +} + +export function prepareProviderAttachments( + files: UserFile[] | undefined, + providerId: ProviderId | string +): PreparedProviderAttachment[] { + if (!files || files.length === 0) return [] + + const provider = getAttachmentProvider(providerId) + if (!provider) { + throw new Error(`File attachments are not supported for provider "${providerId}"`) + } + + if (UNSUPPORTED_FILE_PROVIDERS.has(provider)) { + throw new Error( + `File attachments are not supported for provider "${providerId}" in the current adapter. Supported attachments: ${getProviderSupportedLabel(provider)}.` + ) + } + + const maxBytes = getProviderAttachmentMaxBytes(providerId) + + return files.map((file) => { + const declaredMimeType = inferAttachmentMimeType(file) + const contentType = getContentType(declaredMimeType) + + if (!contentType) { + throw new Error( + `File "${file.name}" has MIME type "${declaredMimeType}", which is not supported by provider "${providerId}". Supported attachments: ${getProviderSupportedLabel(provider)}.` + ) + } + + if (Number.isFinite(file.size) && file.size > maxBytes) { + const sizeMB = (file.size / (1024 * 1024)).toFixed(2) + const maxMB = (maxBytes / (1024 * 1024)).toFixed(0) + throw new Error( + `File "${file.name}" (${sizeMB}MB) exceeds the ${maxMB}MB agent attachment limit for provider "${providerId}"` + ) + } + + if (!file.base64) { + throw new Error(`File "${file.name}" could not be read for provider "${providerId}"`) + } + + const sniffedImageMimeType = contentType === 'image' ? sniffImageMimeType(file.base64) : '' + if (contentType === 'image' && !sniffedImageMimeType) { + throw new Error( + `Image bytes in "${file.name}" are not a supported model image format (declared MIME type "${declaredMimeType}"). Supported image formats: image/jpeg, image/png, image/gif, image/webp.` + ) + } + + const mimeType = sniffedImageMimeType || declaredMimeType + const extension = getAttachmentExtension(file, mimeType) + const attachment = { + file, + filename: file.name, + mimeType, + base64: file.base64, + extension, + contentType, + } + + validateProviderSupport(attachment, provider, providerId) + + const providerMimeType = normalizeProviderMimeType(mimeType, provider) + return { + ...attachment, + providerMimeType, + dataUrl: toDataUrl(providerMimeType, file.base64), + ...(isTextDocumentMimeType(mimeType) && { + text: decodeBase64Text(file.base64, file.name), + }), + } + }) +} + +export function buildOpenAIMessageContent( + content: string | null | undefined, + files: UserFile[] | undefined, + providerId: ProviderId | string +): string | Array> { + const attachments = prepareProviderAttachments(files, providerId) + if (attachments.length === 0) return content ?? '' + + const parts: Array> = [] + if (content) { + parts.push({ type: 'input_text', text: content }) + } + + for (const attachment of attachments) { + if (attachment.contentType === 'image') { + parts.push({ type: 'input_image', image_url: attachment.dataUrl }) + } else { + parts.push({ + type: 'input_file', + filename: attachment.filename, + file_data: attachment.dataUrl, + }) + } + } + + return parts +} + +export function buildAnthropicMessageContent( + content: string | null | undefined, + files: UserFile[] | undefined, + providerId: ProviderId | string +): Array> { + const parts: Array> = [] + if (content) { + parts.push({ type: 'text', text: content }) + } + + for (const attachment of prepareProviderAttachments(files, providerId)) { + if (attachment.contentType === 'image') { + parts.push({ + type: 'image', + source: { + type: 'base64', + media_type: attachment.providerMimeType, + data: attachment.base64, + }, + }) + } else if (attachment.text) { + parts.push({ + type: 'document', + source: { + type: 'text', + media_type: 'text/plain', + data: attachment.text, + }, + title: attachment.filename, + }) + } else { + parts.push({ + type: 'document', + source: { + type: 'base64', + media_type: attachment.providerMimeType, + data: attachment.base64, + }, + title: attachment.filename, + }) + } + } + + return parts +} + +export function buildGeminiMessageParts( + content: string | null | undefined, + files: UserFile[] | undefined, + providerId: ProviderId | string +): Array> { + const parts: Array> = [] + if (content) { + parts.push({ text: content }) + } + + for (const attachment of prepareProviderAttachments(files, providerId)) { + parts.push({ + inlineData: { + mimeType: attachment.providerMimeType, + data: attachment.base64, + }, + }) + } + + return parts +} + +export function buildOpenAICompatibleChatContent( + content: string | null | undefined, + files: UserFile[] | undefined, + providerId: ProviderId | string +): string | Array> { + const attachments = prepareProviderAttachments(files, providerId) + if (attachments.length === 0) return content ?? '' + + const parts: Array> = [] + if (content) { + parts.push({ type: 'text', text: content }) + } + + for (const attachment of attachments) { + parts.push({ + type: 'image_url', + image_url: { + url: attachment.dataUrl, + }, + }) + } + + return parts +} + +export function buildOpenRouterMessageContent( + content: string | null | undefined, + files: UserFile[] | undefined, + providerId: ProviderId | string +): string | Array> { + const attachments = prepareProviderAttachments(files, providerId) + if (attachments.length === 0) return content ?? '' + + const parts: Array> = [] + if (content) { + parts.push({ type: 'text', text: content }) + } + + for (const attachment of attachments) { + if (attachment.contentType === 'image') { + parts.push({ + type: 'image_url', + image_url: { url: attachment.dataUrl }, + }) + } else { + parts.push({ + type: 'file', + file: { + filename: attachment.filename, + file_data: attachment.dataUrl, + }, + }) + } + } + + return parts +} + +function sanitizeBedrockName(filename: string): string { + const baseName = filename.replace(/\.[^/.]+$/, '').replace(/[^a-zA-Z0-9\s()[\]-]/g, ' ') + const compacted = baseName.replace(/\s+/g, ' ').trim() + return compacted || 'Document' +} + +function getBedrockDocumentFormat(attachment: PreparedProviderAttachment): string { + if (attachment.extension === 'md' || attachment.mimeType === 'text/markdown') return 'md' + if (attachment.extension === 'txt' || attachment.mimeType === 'text/plain') return 'txt' + return attachment.extension || 'txt' +} + +function getBedrockImageFormat(attachment: PreparedProviderAttachment): string { + return attachment.extension === 'jpg' ? 'jpeg' : attachment.extension +} + +export function buildBedrockMessageContent( + content: string | null | undefined, + files: UserFile[] | undefined, + providerId: ProviderId | string +): Array> { + const parts: Array> = [] + if (content) { + parts.push({ text: content }) + } + + for (const attachment of prepareProviderAttachments(files, providerId)) { + const bytes = Buffer.from(attachment.base64, 'base64') + if (attachment.contentType === 'image') { + parts.push({ + image: { + format: getBedrockImageFormat(attachment), + source: { bytes }, + }, + }) + } else if (attachment.contentType === 'video') { + parts.push({ + video: { + format: attachment.extension, + source: { bytes }, + }, + }) + } else { + parts.push({ + document: { + format: getBedrockDocumentFormat(attachment), + name: sanitizeBedrockName(attachment.filename), + source: { bytes }, + }, + }) + } + } + + return parts +} + +export function formatMessagesForProvider( + messages: Array<{ role: string; content?: string | null; files?: UserFile[] }>, + providerId: ProviderId | string +) { + return messages.map((message) => { + if (!message.files?.length || (message.role !== 'user' && message.role !== 'assistant')) { + return message + } + + const provider = getAttachmentProvider(providerId) + if (provider === 'openrouter') { + const { files: _files, ...rest } = message + return { + ...rest, + content: buildOpenRouterMessageContent(message.content, message.files, providerId), + } + } + + const { files: _files, ...rest } = message + return { + ...rest, + content: buildOpenAICompatibleChatContent(message.content, message.files, providerId), + } + }) +} diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index a5c9fcd633f..e6bb18655e4 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -90,6 +90,9 @@ async function executeChatCompletionsRequest( } if (request.messages) { + if (request.messages.some((message) => message.files?.length)) { + throw new Error('File attachments require an Azure OpenAI Responses API endpoint') + } allMessages.push(...(request.messages as ChatCompletionMessageParam[])) } diff --git a/apps/sim/providers/bedrock/index.ts b/apps/sim/providers/bedrock/index.ts index 31c8d14cfc6..d4f4dc22619 100644 --- a/apps/sim/providers/bedrock/index.ts +++ b/apps/sim/providers/bedrock/index.ts @@ -17,6 +17,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import type { IterationToolCall, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { buildBedrockMessageContent } from '@/providers/attachments' import { checkForForcedToolUsage, createReadableStreamFromBedrockStream, @@ -180,7 +181,11 @@ export const bedrockProvider: ProviderConfig = { const role: ConversationRole = msg.role === 'assistant' ? 'assistant' : 'user' messages.push({ role, - content: [{ text: msg.content || '' }], + content: buildBedrockMessageContent( + msg.content, + msg.files, + 'bedrock' + ) as unknown as ContentBlock[], }) } } diff --git a/apps/sim/providers/cerebras/index.ts b/apps/sim/providers/cerebras/index.ts index fe6f0bba76c..795cdaa498e 100644 --- a/apps/sim/providers/cerebras/index.ts +++ b/apps/sim/providers/cerebras/index.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' import type { CerebrasResponse } from '@/providers/cerebras/types' import { createReadableStreamFromCerebrasStream } from '@/providers/cerebras/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' @@ -64,6 +65,7 @@ export const cerebrasProvider: ProviderConfig = { if (request.messages) { allMessages.push(...request.messages) } + const formattedMessages = formatMessagesForProvider(allMessages, 'cerebras') const tools = request.tools?.length ? request.tools.map((tool) => ({ @@ -78,7 +80,7 @@ export const cerebrasProvider: ProviderConfig = { const payload: any = { model: request.model.replace('cerebras/', ''), - messages: allMessages, + messages: formattedMessages, } if (request.temperature !== undefined) payload.temperature = request.temperature if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens @@ -199,7 +201,7 @@ export const cerebrasProvider: ProviderConfig = { } const toolCalls = [] const toolResults: Record[] = [] - const currentMessages = [...allMessages] + const currentMessages = [...formattedMessages] let iterationCount = 0 let modelTime = firstResponseTime diff --git a/apps/sim/providers/deepseek/index.ts b/apps/sim/providers/deepseek/index.ts index 6f5c0612e3d..fe06798d728 100644 --- a/apps/sim/providers/deepseek/index.ts +++ b/apps/sim/providers/deepseek/index.ts @@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors' import OpenAI from 'openai' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' import { createReadableStreamFromDeepseekStream } from '@/providers/deepseek/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment' @@ -67,6 +68,7 @@ export const deepseekProvider: ProviderConfig = { if (request.messages) { allMessages.push(...request.messages) } + const formattedMessages = formatMessagesForProvider(allMessages, 'deepseek') const tools = request.tools?.length ? request.tools.map((tool) => ({ @@ -81,7 +83,7 @@ export const deepseekProvider: ProviderConfig = { const payload: any = { model: request.model, - messages: allMessages, + messages: formattedMessages, } if (request.temperature !== undefined) payload.temperature = request.temperature @@ -209,7 +211,7 @@ export const deepseekProvider: ProviderConfig = { } const toolCalls = [] const toolResults: Record[] = [] - const currentMessages = [...allMessages] + const currentMessages = [...formattedMessages] let iterationCount = 0 let hasUsedForcedTool = false let modelTime = firstResponseTime diff --git a/apps/sim/providers/fireworks/index.ts b/apps/sim/providers/fireworks/index.ts index 6aa336ec7b9..5b01076bf07 100644 --- a/apps/sim/providers/fireworks/index.ts +++ b/apps/sim/providers/fireworks/index.ts @@ -4,6 +4,7 @@ import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' import { checkForForcedToolUsage, createReadableStreamFromOpenAIStream, @@ -108,6 +109,7 @@ export const fireworksProvider: ProviderConfig = { if (request.messages) { allMessages.push(...request.messages) } + const formattedMessages = formatMessagesForProvider(allMessages, 'fireworks') as Message[] const tools = request.tools?.length ? request.tools.map((tool) => ({ @@ -122,7 +124,7 @@ export const fireworksProvider: ProviderConfig = { const payload: any = { model: requestedModel, - messages: allMessages, + messages: formattedMessages, } if (request.temperature !== undefined) payload.temperature = request.temperature @@ -250,7 +252,7 @@ export const fireworksProvider: ProviderConfig = { } const toolCalls: FunctionCallResponse[] = [] const toolResults: Record[] = [] - const currentMessages = [...allMessages] + const currentMessages = [...formattedMessages] let iterationCount = 0 let modelTime = firstResponseTime let toolsTime = 0 diff --git a/apps/sim/providers/gemini/core.ts b/apps/sim/providers/gemini/core.ts index 511a8df7ad7..6536d6336f6 100644 --- a/apps/sim/providers/gemini/core.ts +++ b/apps/sim/providers/gemini/core.ts @@ -917,7 +917,7 @@ export async function executeGeminiRequest( const providerStartTimeISO = new Date(providerStartTime).toISOString() try { - const { contents, tools, systemInstruction } = convertToGeminiFormat(request) + const { contents, tools, systemInstruction } = convertToGeminiFormat(request, providerType) // Build configuration const geminiConfig: GenerateContentConfig = {} diff --git a/apps/sim/providers/google/utils.test.ts b/apps/sim/providers/google/utils.test.ts index 31d430e2312..2b6a4e4d0cf 100644 --- a/apps/sim/providers/google/utils.test.ts +++ b/apps/sim/providers/google/utils.test.ts @@ -111,6 +111,44 @@ describe('ensureStructResponse', () => { }) describe('convertToGeminiFormat', () => { + it('should convert user message files to inline data parts', () => { + const request: ProviderRequest = { + model: 'gemini-2.5-flash', + messages: [ + { + role: 'user', + content: 'Analyze this image', + files: [ + { + id: 'file-1', + key: 'workspace/ws-1/example.png', + name: 'example.png', + url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace', + size: 128, + type: 'image/png', + base64: 'aW1hZ2U=', + }, + ], + }, + ], + } + + const result = convertToGeminiFormat(request) + + expect(result.contents[0]).toEqual({ + role: 'user', + parts: [ + { text: 'Analyze this image' }, + { + inlineData: { + mimeType: 'image/png', + data: 'aW1hZ2U=', + }, + }, + ], + }) + }) + describe('tool message handling', () => { it('should convert tool message with object response correctly', () => { const request: ProviderRequest = { diff --git a/apps/sim/providers/google/utils.ts b/apps/sim/providers/google/utils.ts index 3f6e37ae927..089f7dac552 100644 --- a/apps/sim/providers/google/utils.ts +++ b/apps/sim/providers/google/utils.ts @@ -14,6 +14,7 @@ import { } from '@google/genai' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { buildGeminiMessageParts } from '@/providers/attachments' import type { ProviderRequest } from '@/providers/types' import { trackForcedToolUsage } from '@/providers/utils' @@ -166,7 +167,10 @@ export interface GeminiToolDef { /** * Converts OpenAI-style request format to Gemini format */ -export function convertToGeminiFormat(request: ProviderRequest): { +export function convertToGeminiFormat( + request: ProviderRequest, + providerId = 'google' +): { contents: Content[] tools: GeminiToolDef[] | undefined systemInstruction: Content | undefined @@ -192,9 +196,10 @@ export function convertToGeminiFormat(request: ProviderRequest): { } } else if (message.role === 'user' || message.role === 'assistant') { const geminiRole = message.role === 'user' ? 'user' : 'model' + const parts = buildGeminiMessageParts(message.content, message.files, providerId) as Part[] - if (message.content) { - contents.push({ role: geminiRole, parts: [{ text: message.content }] }) + if (parts.length > 0) { + contents.push({ role: geminiRole, parts }) } if (message.role === 'assistant' && message.tool_calls?.length) { diff --git a/apps/sim/providers/groq/index.ts b/apps/sim/providers/groq/index.ts index 192e1412d94..4165724bec8 100644 --- a/apps/sim/providers/groq/index.ts +++ b/apps/sim/providers/groq/index.ts @@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors' import { Groq } from 'groq-sdk' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' import { createReadableStreamFromGroqStream } from '@/providers/groq/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment' @@ -60,6 +61,7 @@ export const groqProvider: ProviderConfig = { if (request.messages) { allMessages.push(...request.messages) } + const formattedMessages = formatMessagesForProvider(allMessages, 'groq') const tools = request.tools?.length ? request.tools.map((tool) => ({ @@ -74,7 +76,7 @@ export const groqProvider: ProviderConfig = { const payload: any = { model: request.model.replace('groq/', ''), - messages: allMessages, + messages: formattedMessages, } if (request.temperature !== undefined) payload.temperature = request.temperature @@ -205,7 +207,7 @@ export const groqProvider: ProviderConfig = { } const toolCalls = [] const toolResults: Record[] = [] - const currentMessages = [...allMessages] + const currentMessages = [...formattedMessages] let iterationCount = 0 let modelTime = firstResponseTime let toolsTime = 0 diff --git a/apps/sim/providers/mistral/index.ts b/apps/sim/providers/mistral/index.ts index ffe1ecad930..ce8959f3048 100644 --- a/apps/sim/providers/mistral/index.ts +++ b/apps/sim/providers/mistral/index.ts @@ -4,6 +4,7 @@ import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' import { createReadableStreamFromMistralStream } from '@/providers/mistral/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment' @@ -77,6 +78,7 @@ export const mistralProvider: ProviderConfig = { if (request.messages) { allMessages.push(...request.messages) } + const formattedMessages = formatMessagesForProvider(allMessages, 'mistral') const tools = request.tools?.length ? request.tools.map((tool) => ({ @@ -91,7 +93,7 @@ export const mistralProvider: ProviderConfig = { const payload: any = { model: request.model, - messages: allMessages, + messages: formattedMessages, } if (request.temperature !== undefined) payload.temperature = request.temperature @@ -262,7 +264,7 @@ export const mistralProvider: ProviderConfig = { } const toolCalls = [] const toolResults: Record[] = [] - const currentMessages = [...allMessages] + const currentMessages = [...formattedMessages] let iterationCount = 0 let modelTime = firstResponseTime diff --git a/apps/sim/providers/ollama/index.ts b/apps/sim/providers/ollama/index.ts index 045dd1d462a..111956140c8 100644 --- a/apps/sim/providers/ollama/index.ts +++ b/apps/sim/providers/ollama/index.ts @@ -5,6 +5,7 @@ import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/ import { getOllamaUrl } from '@/lib/core/utils/urls' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' import type { ModelsObject } from '@/providers/ollama/types' import { createReadableStreamFromOllamaStream } from '@/providers/ollama/utils' import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment' @@ -91,6 +92,7 @@ export const ollamaProvider: ProviderConfig = { if (request.messages) { allMessages.push(...request.messages) } + const formattedMessages = formatMessagesForProvider(allMessages, 'ollama') const tools = request.tools?.length ? request.tools.map((tool) => ({ @@ -105,7 +107,7 @@ export const ollamaProvider: ProviderConfig = { const payload: any = { model: request.model, - messages: allMessages, + messages: formattedMessages, } if (request.temperature !== undefined) payload.temperature = request.temperature @@ -274,7 +276,7 @@ export const ollamaProvider: ProviderConfig = { } const toolCalls = [] const toolResults: Record[] = [] - const currentMessages = [...allMessages] + const currentMessages = [...formattedMessages] let iterationCount = 0 let modelTime = firstResponseTime diff --git a/apps/sim/providers/openai/core.ts b/apps/sim/providers/openai/core.ts index 1f025269235..1b2591d22e2 100644 --- a/apps/sim/providers/openai/core.ts +++ b/apps/sim/providers/openai/core.ts @@ -133,7 +133,7 @@ export async function executeResponsesProviderRequest( allMessages.push(...request.messages) } - const initialInput = buildResponsesInputFromMessages(allMessages) + const initialInput = buildResponsesInputFromMessages(allMessages, config.providerId) const basePayload: Record = { model: config.modelName, diff --git a/apps/sim/providers/openai/utils.test.ts b/apps/sim/providers/openai/utils.test.ts new file mode 100644 index 00000000000..868604b7694 --- /dev/null +++ b/apps/sim/providers/openai/utils.test.ts @@ -0,0 +1,40 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { buildResponsesInputFromMessages } from '@/providers/openai/utils' + +describe('buildResponsesInputFromMessages', () => { + it('should convert user message files to Responses multipart content', () => { + const input = buildResponsesInputFromMessages([ + { + role: 'user', + content: 'Analyze this image', + files: [ + { + id: 'file-1', + key: 'workspace/ws-1/example.png', + name: 'example.png', + url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace', + size: 128, + type: 'image/png', + base64: 'aW1hZ2U=', + }, + ], + }, + ]) + + expect(input).toEqual([ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Analyze this image' }, + { + type: 'input_image', + image_url: 'data:image/png;base64,aW1hZ2U=', + }, + ], + }, + ]) + }) +}) diff --git a/apps/sim/providers/openai/utils.ts b/apps/sim/providers/openai/utils.ts index a1edfd9eae2..ad6a5215c19 100644 --- a/apps/sim/providers/openai/utils.ts +++ b/apps/sim/providers/openai/utils.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import type OpenAI from 'openai' +import { buildOpenAIMessageContent } from '@/providers/attachments' import type { Message } from '@/providers/types' const logger = createLogger('ResponsesUtils') @@ -21,7 +22,7 @@ export interface ResponsesToolCall { export type ResponsesInputItem = | { role: 'system' | 'user' | 'assistant' - content: string + content: string | Array> } | { type: 'function_call' @@ -45,7 +46,10 @@ export interface ResponsesToolDefinition { /** * Converts chat-style messages into Responses API input items. */ -export function buildResponsesInputFromMessages(messages: Message[]): ResponsesInputItem[] { +export function buildResponsesInputFromMessages( + messages: Message[], + providerId = 'openai' +): ResponsesInputItem[] { const input: ResponsesInputItem[] = [] for (const message of messages) { @@ -58,13 +62,21 @@ export function buildResponsesInputFromMessages(messages: Message[]): ResponsesI continue } - if ( - message.content && - (message.role === 'system' || message.role === 'user' || message.role === 'assistant') - ) { + if (message.role === 'system' || message.role === 'user' || message.role === 'assistant') { + const content = + message.role === 'user' + ? buildOpenAIMessageContent(message.content, message.files, providerId) + : (message.content ?? '') + if ( + (typeof content === 'string' && !content) || + (Array.isArray(content) && content.length === 0) + ) { + continue + } + input.push({ role: message.role, - content: message.content, + content, }) } diff --git a/apps/sim/providers/openrouter/index.ts b/apps/sim/providers/openrouter/index.ts index 87ff07fcfef..09a8adbbdd5 100644 --- a/apps/sim/providers/openrouter/index.ts +++ b/apps/sim/providers/openrouter/index.ts @@ -4,6 +4,7 @@ import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { checkForForcedToolUsage, @@ -109,6 +110,7 @@ export const openRouterProvider: ProviderConfig = { if (request.messages) { allMessages.push(...request.messages) } + const formattedMessages = formatMessagesForProvider(allMessages, 'openrouter') as Message[] const tools = request.tools?.length ? request.tools.map((tool) => ({ @@ -123,7 +125,7 @@ export const openRouterProvider: ProviderConfig = { const payload: any = { model: requestedModel, - messages: allMessages, + messages: formattedMessages, } if (request.temperature !== undefined) payload.temperature = request.temperature @@ -251,7 +253,7 @@ export const openRouterProvider: ProviderConfig = { } const toolCalls: FunctionCallResponse[] = [] const toolResults: Record[] = [] - const currentMessages = [...allMessages] + const currentMessages = [...formattedMessages] let iterationCount = 0 let modelTime = firstResponseTime let toolsTime = 0 diff --git a/apps/sim/providers/types.ts b/apps/sim/providers/types.ts index 9ce73bffcef..007b9b3ead5 100644 --- a/apps/sim/providers/types.ts +++ b/apps/sim/providers/types.ts @@ -1,4 +1,4 @@ -import type { ProviderTimingSegment, StreamingExecution } from '@/executor/types' +import type { ProviderTimingSegment, StreamingExecution, UserFile } from '@/executor/types' export type ProviderId = | 'openai' @@ -121,6 +121,7 @@ export interface ProviderToolConfig { export interface Message { role: 'system' | 'user' | 'assistant' | 'function' | 'tool' content: string | null + files?: UserFile[] name?: string function_call?: { name: string diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index db25ba45ec0..b182eefcd91 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -5,6 +5,7 @@ import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/ import { env } from '@/lib/core/config/env' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment' import type { @@ -123,6 +124,7 @@ export const vllmProvider: ProviderConfig = { if (request.messages) { allMessages.push(...request.messages) } + const formattedMessages = formatMessagesForProvider(allMessages, 'vllm') as Message[] const tools = request.tools?.length ? request.tools.map((tool) => ({ @@ -137,7 +139,7 @@ export const vllmProvider: ProviderConfig = { const payload: any = { model: request.model.replace(/^vllm\//, ''), - messages: allMessages, + messages: formattedMessages, } if (request.temperature !== undefined) payload.temperature = request.temperature @@ -319,7 +321,7 @@ export const vllmProvider: ProviderConfig = { } const toolCalls = [] const toolResults: Record[] = [] - const currentMessages = [...allMessages] + const currentMessages = [...formattedMessages] let iterationCount = 0 let modelTime = firstResponseTime diff --git a/apps/sim/providers/xai/index.ts b/apps/sim/providers/xai/index.ts index 309a9fd8f3b..bd6303007df 100644 --- a/apps/sim/providers/xai/index.ts +++ b/apps/sim/providers/xai/index.ts @@ -4,6 +4,7 @@ import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment' import type { @@ -76,6 +77,7 @@ export const xAIProvider: ProviderConfig = { if (request.messages) { allMessages.push(...request.messages) } + const formattedMessages = formatMessagesForProvider(allMessages, 'xai') as Message[] const tools = request.tools?.length ? request.tools.map((tool) => ({ type: 'function', @@ -93,7 +95,7 @@ export const xAIProvider: ProviderConfig = { } const basePayload: any = { model: request.model, - messages: allMessages, + messages: formattedMessages, } if (request.temperature !== undefined) basePayload.temperature = request.temperature @@ -219,7 +221,7 @@ export const xAIProvider: ProviderConfig = { } const toolCalls = [] const toolResults: Record[] = [] - const currentMessages = [...allMessages] + const currentMessages = [...formattedMessages] let iterationCount = 0 let hasUsedForcedTool = false From 488061433bfde2963f72eb9d66c9fdf77c810bbc Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan Date: Thu, 14 May 2026 18:21:38 -0700 Subject: [PATCH 3/6] Clean up attachments --- apps/sim/providers/attachments.ts | 97 ++++++------------------- apps/sim/providers/google/utils.test.ts | 4 +- apps/sim/providers/openai/utils.test.ts | 4 +- 3 files changed, 27 insertions(+), 78 deletions(-) diff --git a/apps/sim/providers/attachments.ts b/apps/sim/providers/attachments.ts index eda2b552b48..e84daf4b2a5 100644 --- a/apps/sim/providers/attachments.ts +++ b/apps/sim/providers/attachments.ts @@ -1,7 +1,9 @@ import { getContentType, + getExtensionFromMimeType, getFileExtension, getMimeTypeFromExtension, + MIME_TYPE_MAPPING, MODEL_SUPPORTED_IMAGE_MIME_TYPES, } from '@/lib/uploads/utils/file-utils' import type { UserFile } from '@/executor/types' @@ -37,64 +39,15 @@ export interface PreparedProviderAttachment { const AGENT_ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024 const PDF_MIME_TYPE = 'application/pdf' -const TEXT_DOCUMENT_MIME_TYPES = new Set([ - 'text/plain', - 'text/markdown', - 'text/csv', - 'text/html', - 'text/xml', - 'text/javascript', - 'text/typescript', - 'text/x-python', - 'text/x-go', - 'text/x-rust', - 'text/x-java', - 'text/x-kotlin', - 'text/x-c', - 'text/x-c++', - 'text/x-csharp', - 'text/x-ruby', - 'text/x-php', - 'text/x-swift', - 'text/x-shellscript', - 'application/json', - 'application/xml', - 'application/x-yaml', -]) +const DOCUMENT_MIME_TYPES = new Set( + Object.entries(MIME_TYPE_MAPPING) + .filter(([, contentType]) => contentType === 'document') + .map(([mimeType]) => mimeType) +) -const OPENAI_DOCUMENT_MIME_TYPES = new Set([ - PDF_MIME_TYPE, - ...TEXT_DOCUMENT_MIME_TYPES, - 'application/rtf', - 'application/msword', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'application/vnd.ms-powerpoint', - 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - 'application/vnd.ms-excel', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', -]) +const OPENAI_DOCUMENT_MIME_TYPES = new Set([...DOCUMENT_MIME_TYPES, 'application/x-yaml']) -const GEMINI_INLINE_MIME_TYPES = new Set([ - PDF_MIME_TYPE, - ...TEXT_DOCUMENT_MIME_TYPES, - ...MODEL_SUPPORTED_IMAGE_MIME_TYPES, - 'audio/mpeg', - 'audio/mp3', - 'audio/mp4', - 'audio/x-m4a', - 'audio/m4a', - 'audio/wav', - 'audio/wave', - 'audio/x-wav', - 'audio/webm', - 'audio/ogg', - 'audio/flac', - 'video/mp4', - 'video/mpeg', - 'video/quicktime', - 'video/x-quicktime', - 'video/webm', -]) +const GEMINI_INLINE_MIME_TYPES = new Set([...Object.keys(MIME_TYPE_MAPPING), 'application/x-yaml']) const BEDROCK_DOCUMENT_FORMATS = new Set([ 'pdf', @@ -168,7 +121,12 @@ export function inferAttachmentMimeType(file: UserFile): string { } function isTextDocumentMimeType(mimeType: string): boolean { - return TEXT_DOCUMENT_MIME_TYPES.has(mimeType) || mimeType.startsWith('text/') + return ( + mimeType.startsWith('text/') || + mimeType === 'application/json' || + mimeType === 'application/xml' || + mimeType === 'application/x-yaml' + ) } function isImageMimeType(mimeType: string): boolean { @@ -179,6 +137,12 @@ function isOpenAIDocumentMimeType(mimeType: string): boolean { return OPENAI_DOCUMENT_MIME_TYPES.has(mimeType) || isTextDocumentMimeType(mimeType) } +function getAttachmentContentType( + mimeType: string +): PreparedProviderAttachment['contentType'] | null { + return getContentType(mimeType) || (isTextDocumentMimeType(mimeType) ? 'document' : null) +} + function sniffImageMimeType(base64: string): string { let bytes: Buffer try { @@ -218,23 +182,8 @@ function sniffImageMimeType(base64: string): string { } function getAttachmentExtension(file: UserFile, mimeType: string): string { - if (mimeType === 'image/jpeg' || mimeType === 'image/jpg') return 'jpeg' - if (mimeType === 'image/png') return 'png' - if (mimeType === 'image/gif') return 'gif' - if (mimeType === 'image/webp') return 'webp' - if (mimeType === 'video/mp4') return 'mp4' - if (mimeType === 'video/quicktime' || mimeType === 'video/x-quicktime') return 'mov' - if (mimeType === 'video/webm') return 'webm' - - const extension = getFileExtension(file.name) - if (extension) return extension - - if (mimeType === 'application/pdf') return 'pdf' if (mimeType === 'text/markdown') return 'md' - if (mimeType === 'text/plain') return 'txt' - if (mimeType === 'text/csv') return 'csv' - if (mimeType === 'text/html') return 'html' - return '' + return getExtensionFromMimeType(mimeType) || getFileExtension(file.name) } function normalizeProviderMimeType(mimeType: string, provider: AttachmentProvider): string { @@ -315,7 +264,7 @@ export function prepareProviderAttachments( return files.map((file) => { const declaredMimeType = inferAttachmentMimeType(file) - const contentType = getContentType(declaredMimeType) + const contentType = getAttachmentContentType(declaredMimeType) if (!contentType) { throw new Error( diff --git a/apps/sim/providers/google/utils.test.ts b/apps/sim/providers/google/utils.test.ts index 2b6a4e4d0cf..7489216049a 100644 --- a/apps/sim/providers/google/utils.test.ts +++ b/apps/sim/providers/google/utils.test.ts @@ -126,7 +126,7 @@ describe('convertToGeminiFormat', () => { url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace', size: 128, type: 'image/png', - base64: 'aW1hZ2U=', + base64: 'iVBORw0KGgo=', }, ], }, @@ -142,7 +142,7 @@ describe('convertToGeminiFormat', () => { { inlineData: { mimeType: 'image/png', - data: 'aW1hZ2U=', + data: 'iVBORw0KGgo=', }, }, ], diff --git a/apps/sim/providers/openai/utils.test.ts b/apps/sim/providers/openai/utils.test.ts index 868604b7694..20fcd484ad8 100644 --- a/apps/sim/providers/openai/utils.test.ts +++ b/apps/sim/providers/openai/utils.test.ts @@ -18,7 +18,7 @@ describe('buildResponsesInputFromMessages', () => { url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace', size: 128, type: 'image/png', - base64: 'aW1hZ2U=', + base64: 'iVBORw0KGgo=', }, ], }, @@ -31,7 +31,7 @@ describe('buildResponsesInputFromMessages', () => { { type: 'input_text', text: 'Analyze this image' }, { type: 'input_image', - image_url: 'data:image/png;base64,aW1hZ2U=', + image_url: 'data:image/png;base64,iVBORw0KGgo=', }, ], }, From c9f0dca7041beeacd28993b1aae31a86468c5cd8 Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan Date: Thu, 14 May 2026 18:54:49 -0700 Subject: [PATCH 4/6] Fix start.files --- .../app/api/workflows/[id]/execute/route.ts | 4 +- .../hooks/use-workflow-execution.ts | 3 + .../utils/workflow-execution-utils.ts | 1 + .../executor/handlers/agent/agent-handler.ts | 93 ++++++++++++++----- .../executor/handlers/agent/memory.test.ts | 28 ++++++ apps/sim/executor/handlers/agent/memory.ts | 32 +++++-- apps/sim/executor/utils/start-block.test.ts | 36 +++++++ apps/sim/executor/utils/start-block.ts | 67 ++++++++++++- apps/sim/hooks/use-execution-stream.ts | 1 + apps/sim/lib/api/contracts/workflows.ts | 1 + .../uploads/utils/user-file-base64.server.ts | 8 +- apps/sim/providers/attachments.ts | 5 + 12 files changed, 241 insertions(+), 38 deletions(-) diff --git a/apps/sim/app/api/workflows/[id]/execute/route.ts b/apps/sim/app/api/workflows/[id]/execute/route.ts index 9d042cea756..b2d80e8ddb2 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.ts @@ -359,6 +359,7 @@ async function handleExecutePost( includeFileBase64, base64MaxBytes, workflowStateOverride, + executionId: requestedExecutionId, triggerBlockId: parsedTriggerBlockId, startBlockId, stopAfterBlockId, @@ -508,7 +509,8 @@ async function handleExecutePost( ) } - const executionId = generateId() + const executionId = + isClientSession && requestedExecutionId ? requestedExecutionId : generateId() reqLogger = reqLogger.withMetadata({ userId, executionId }) reqLogger.info('Starting server-side execution', { diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts index 845801846d2..604d0edd9c4 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts @@ -524,6 +524,7 @@ export function useWorkflowExecution() { size: fileData.file.size, type: fileData.file.type, key: result.key, + context: 'execution', }) } catch (uploadError) { if ( @@ -565,6 +566,7 @@ export function useWorkflowExecution() { size: r.size, type: r.type, key: r.key, + context: r.context || 'execution', uploadedAt: r.uploadedAt, expiresAt: r.expiresAt, }) @@ -1126,6 +1128,7 @@ export function useWorkflowExecution() { await executionStream.execute({ workflowId: activeWorkflowId, input: finalWorkflowInput, + executionId, startBlockId, selectedOutputs, triggerType: overrideTriggerType || 'manual', diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts index 4b9e726d089..6e26e92fe81 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts @@ -903,6 +903,7 @@ export async function executeWorkflowWithFullLogging( triggerType: options.overrideTriggerType || 'manual', useDraftState: options.useDraftState ?? true, isClientSession: true, + ...(options.executionId ? { executionId: options.executionId } : {}), ...(options.triggerBlockId ? { triggerBlockId: options.triggerBlockId } : {}), ...(options.stopAfterBlockId ? { stopAfterBlockId: options.stopAfterBlockId } : {}), ...(options.runFromBlock diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 08a95be5195..b774dcfcf28 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -39,7 +39,7 @@ import { buildAPIUrl, buildAuthHeaders } from '@/executor/utils/http' import { stringifyJSON } from '@/executor/utils/json' import { resolveVertexCredential } from '@/executor/utils/vertex-credential' import { executeProviderRequest } from '@/providers' -import { getAttachmentProvider, getProviderAttachmentMaxBytes } from '@/providers/attachments' +import { getProviderAttachmentMaxBytes, supportsFileAttachments } from '@/providers/attachments' import { getProviderFromModel, transformBlockTool } from '@/providers/utils' import type { SerializedBlock } from '@/serializer/types' import { filterSchemaForLLM, type ToolSchema } from '@/tools/params' @@ -91,10 +91,14 @@ export class AgentBlockHandler implements BlockHandler { const streamingConfig = this.getStreamingConfig(ctx, block) const messages = await this.buildMessages(ctx, filteredInputs, skillMetadata) - const messagesWithFiles = await this.attachFilesToLastUserMessage( + const messagesWithInputFiles = this.attachFilesToLastUserMessage( ctx, messages, - filteredInputs.files, + filteredInputs.files + ) + const messagesWithFiles = await this.hydrateMessageFilesForProvider( + ctx, + messagesWithInputFiles, providerId ) @@ -682,12 +686,11 @@ export class AgentBlockHandler implements BlockHandler { return messages.length > 0 ? messages : undefined } - private async attachFilesToLastUserMessage( + private attachFilesToLastUserMessage( ctx: ExecutionContext, messages: Message[] | undefined, - filesInput: unknown, - providerId: string - ): Promise { + filesInput: unknown + ): Message[] | undefined { const normalizedFiles = normalizeFileInput(filesInput) if (!normalizedFiles || normalizedFiles.length === 0) { return messages @@ -697,10 +700,6 @@ export class AgentBlockHandler implements BlockHandler { throw new Error('Files require at least one user message in the agent prompt') } - if (!getAttachmentProvider(providerId)) { - throw new Error(`File attachments are not supported for provider "${providerId}"`) - } - let lastUserMessageIndex = -1 for (let index = messages.length - 1; index >= 0; index--) { if (messages[index].role === 'user') { @@ -718,21 +717,71 @@ export class AgentBlockHandler implements BlockHandler { throw new Error('Files must include at least one valid file object') } - const hydratedFiles = await hydrateUserFilesWithBase64(userFiles, { - requestId, - workspaceId: ctx.workspaceId, - workflowId: ctx.workflowId, - executionId: ctx.executionId, - userId: ctx.userId, - logger, - maxBytes: getProviderAttachmentMaxBytes(providerId), - }) - const lastUserMessage = messages[lastUserMessageIndex] const nextMessages = [...messages] nextMessages[lastUserMessageIndex] = { ...lastUserMessage, - files: [...(lastUserMessage.files ?? []), ...hydratedFiles], + files: [...(lastUserMessage.files ?? []), ...userFiles], + } + + return nextMessages + } + + private async hydrateMessageFilesForProvider( + ctx: ExecutionContext, + messages: Message[] | undefined, + providerId: string + ): Promise { + if (!messages?.some((message) => message.files?.length)) { + return messages + } + + if (!supportsFileAttachments(providerId)) { + throw new Error(`File attachments are not supported for provider "${providerId}"`) + } + + const requestId = ctx.executionId || ctx.workflowId || 'agent-files' + const nextMessages = [...messages] + + for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) { + const message = messages[messageIndex] + const normalizedFiles = normalizeFileInput(message.files) + if (!normalizedFiles || normalizedFiles.length === 0) { + continue + } + + const userFiles = processFilesToUserFiles( + normalizedFiles as RawFileInput[], + requestId, + logger + ) + if (userFiles.length === 0) { + throw new Error('Files must include at least one valid file object') + } + + const hydratedFiles = await hydrateUserFilesWithBase64(userFiles, { + requestId, + workspaceId: ctx.workspaceId, + workflowId: ctx.workflowId, + executionId: ctx.executionId, + largeValueExecutionIds: ctx.largeValueExecutionIds, + allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope, + userId: ctx.userId, + logger, + maxBytes: getProviderAttachmentMaxBytes(providerId), + }) + + const missingFile = hydratedFiles.find((file) => !file.base64) + if (missingFile) { + throw new Error( + `File "${missingFile.name}" could not be read for provider "${providerId}". Make sure the file is still accessible and under the provider attachment size limit.` + ) + } + + nextMessages[messageIndex] = { + ...message, + files: hydratedFiles, + } } return nextMessages diff --git a/apps/sim/executor/handlers/agent/memory.test.ts b/apps/sim/executor/handlers/agent/memory.test.ts index 316b2a0e731..b2182921ce6 100644 --- a/apps/sim/executor/handlers/agent/memory.test.ts +++ b/apps/sim/executor/handlers/agent/memory.test.ts @@ -178,6 +178,34 @@ describe('Memory', () => { }) }) + describe('sanitizeMessageForStorage', () => { + it('should strip file payloads and provider-only fields before memory persistence', () => { + const message: Message = { + role: 'user', + content: 'Analyze this file', + executionId: 'exec-1', + files: [ + { + id: 'file-1', + key: 'workspace/ws-1/example.png', + name: 'example.png', + url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace', + size: 128, + type: 'image/png', + base64: 'iVBORw0KGgo=', + }, + ], + tool_calls: [{ id: 'call-1' }], + } + + expect((memoryService as any).sanitizeMessageForStorage(message)).toEqual({ + role: 'user', + content: 'Analyze this file', + executionId: 'exec-1', + }) + }) + }) + describe('Token-based vs Message-based comparison', () => { it('should produce different results for same limit concept', () => { const messages: Message[] = [ diff --git a/apps/sim/executor/handlers/agent/memory.ts b/apps/sim/executor/handlers/agent/memory.ts index fcba3628226..d9e693210d8 100644 --- a/apps/sim/executor/handlers/agent/memory.ts +++ b/apps/sim/executor/handlers/agent/memory.ts @@ -122,6 +122,14 @@ export class Memory { return messages.slice(-limit) } + private sanitizeMessageForStorage(message: Message): Message { + return { + role: message.role, + content: message.content, + ...(message.executionId && { executionId: message.executionId }), + } + } + private applyTokenWindow(messages: Message[], maxTokens: number, model?: string): Message[] { const result: Message[] = [] let tokenCount = 0 @@ -177,9 +185,17 @@ export class Memory { const data = result[0].data if (!Array.isArray(data)) return [] - return data.filter( - (msg): msg is Message => msg && typeof msg === 'object' && 'role' in msg && 'content' in msg - ) + return data + .filter( + (msg): msg is Message => + msg && + typeof msg === 'object' && + 'role' in msg && + 'content' in msg && + ['system', 'user', 'assistant'].includes(msg.role) && + typeof msg.content === 'string' + ) + .map((msg) => this.sanitizeMessageForStorage(msg)) } private async seedMemoryRecord( @@ -189,13 +205,15 @@ export class Memory { ): Promise { const now = new Date() + const sanitizedMessages = messages.map((message) => this.sanitizeMessageForStorage(message)) + await db .insert(memory) .values({ id: generateId(), workspaceId, key, - data: messages, + data: sanitizedMessages, createdAt: now, updatedAt: now, }) @@ -205,20 +223,22 @@ export class Memory { private async appendMessage(workspaceId: string, key: string, message: Message): Promise { const now = new Date() + const sanitizedMessage = this.sanitizeMessageForStorage(message) + await db .insert(memory) .values({ id: generateId(), workspaceId, key, - data: [message], + data: [sanitizedMessage], createdAt: now, updatedAt: now, }) .onConflictDoUpdate({ target: [memory.workspaceId, memory.key], set: { - data: sql`${memory.data} || ${JSON.stringify([message])}::jsonb`, + data: sql`${memory.data} || ${JSON.stringify([sanitizedMessage])}::jsonb`, updatedAt: now, }, }) diff --git a/apps/sim/executor/utils/start-block.test.ts b/apps/sim/executor/utils/start-block.test.ts index f50c5a58da5..1e8cadf9c48 100644 --- a/apps/sim/executor/utils/start-block.test.ts +++ b/apps/sim/executor/utils/start-block.test.ts @@ -119,6 +119,42 @@ describe('start-block utilities', () => { expect(output.files).toEqual(files) }) + it.concurrent('buildStartBlockOutput normalizes Start files from internal serve URLs', () => { + const block = createBlock('start_trigger', 'start') + const resolution = { + blockId: 'start', + block, + path: StartBlockPath.UNIFIED, + } as const + + const output = buildStartBlockOutput({ + resolution, + workflowInput: { + files: [ + { + id: 'file_1', + name: 'screenshot.png', + url: '/api/files/serve/s3/execution%2Fworkspace-id%2Fworkflow-id%2Fexecution-id%2Fscreenshot.png?context=execution', + size: 243289, + type: 'image/png', + }, + ], + }, + }) + + expect(output.files).toEqual([ + { + id: 'file_1', + name: 'screenshot.png', + url: '/api/files/serve/s3/execution%2Fworkspace-id%2Fworkflow-id%2Fexecution-id%2Fscreenshot.png?context=execution', + size: 243289, + type: 'image/png', + key: 'execution/workspace-id/workflow-id/execution-id/screenshot.png', + context: 'execution', + }, + ]) + }) + it.concurrent('rejects inputFormat fields that collide with executor routing keys', () => { const block = createBlock('start_trigger', 'start', { subBlocks: { diff --git a/apps/sim/executor/utils/start-block.ts b/apps/sim/executor/utils/start-block.ts index 8d0a4afcb0e..ddd75ce221c 100644 --- a/apps/sim/executor/utils/start-block.ts +++ b/apps/sim/executor/utils/start-block.ts @@ -1,4 +1,8 @@ -import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' +import { + inferContextFromKey, + isInternalFileUrl, + parseInternalFileUrl, +} from '@/lib/uploads/utils/file-utils' import { classifyStartBlockType, resolveStartCandidates, @@ -323,13 +327,68 @@ function getRawInputCandidate(workflowInput: unknown): unknown { return workflowInput } +function normalizeStartFile(file: unknown): UserFile | null { + if (!isPlainObject(file)) { + return null + } + + const id = typeof file.id === 'string' ? file.id : '' + const name = typeof file.name === 'string' ? file.name : '' + const url = + typeof file.url === 'string' ? file.url : typeof file.path === 'string' ? file.path : '' + const size = typeof file.size === 'number' ? file.size : Number.NaN + const type = typeof file.type === 'string' ? file.type : '' + const explicitKey = typeof file.key === 'string' ? file.key : '' + + let key = explicitKey + let context = typeof file.context === 'string' ? file.context : undefined + + if (!key && url && isInternalFileUrl(url)) { + try { + const parsed = parseInternalFileUrl(url) + key = parsed.key + context = context || parsed.context + } catch { + return null + } + } + + if (!context && key) { + try { + context = inferContextFromKey(key) + } catch { + // Older file outputs may have opaque keys; keep the file shape intact. + } + } + + if (!id || !name || !url || !Number.isFinite(size) || !type || !key) { + return null + } + + return { + id, + name, + url, + size, + type, + key, + ...(context && { context }), + ...(typeof file.base64 === 'string' && { base64: file.base64 }), + } +} + function getFilesFromWorkflowInput(workflowInput: unknown): UserFile[] | undefined { if (!isPlainObject(workflowInput)) { return undefined } const files = workflowInput.files - if (Array.isArray(files) && files.every(isUserFileWithMetadata)) { - return files + if (!Array.isArray(files)) { + return undefined + } + + const normalizedFiles = files.map(normalizeStartFile) + if (normalizedFiles.every((file): file is UserFile => Boolean(file))) { + return normalizedFiles } return undefined } @@ -341,6 +400,8 @@ function mergeFilesIntoOutput( const files = getFilesFromWorkflowInput(workflowInput) if (files) { output.files = files + } else if (isPlainObject(workflowInput) && Object.hasOwn(workflowInput, 'files')) { + output.files = undefined } return output } diff --git a/apps/sim/hooks/use-execution-stream.ts b/apps/sim/hooks/use-execution-stream.ts index b45a8550ba6..c07fd2a3bff 100644 --- a/apps/sim/hooks/use-execution-stream.ts +++ b/apps/sim/hooks/use-execution-stream.ts @@ -197,6 +197,7 @@ export interface ExecuteStreamOptions { envVarValues?: Record workflowVariables?: Record selectedOutputs?: string[] + executionId?: string startBlockId?: string triggerType?: string useDraftState?: boolean diff --git a/apps/sim/lib/api/contracts/workflows.ts b/apps/sim/lib/api/contracts/workflows.ts index 46e5095c933..e439ff78cdf 100644 --- a/apps/sim/lib/api/contracts/workflows.ts +++ b/apps/sim/lib/api/contracts/workflows.ts @@ -336,6 +336,7 @@ export const executeWorkflowBodySchema = z.object({ includeFileBase64: z.boolean().optional().default(true), base64MaxBytes: z.number().int().positive().optional(), workflowStateOverride: workflowStateSchema.optional(), + executionId: z.string().optional(), triggerBlockId: z.string().optional(), startBlockId: z.string().optional(), stopAfterBlockId: z.string().optional(), diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.ts index 8d5e7b048d1..3f666c16ae6 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.ts @@ -361,8 +361,7 @@ async function resolveBase64( options: Base64HydrationOptions, logger: Logger ): Promise { - const requestedMaxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES - const maxBytes = Math.min(requestedMaxBytes, DEFAULT_MAX_BASE64_BYTES) + const maxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES if (file.base64) { const base64Bytes = Buffer.byteLength(file.base64, 'base64') @@ -440,10 +439,7 @@ async function hydrateUserFile( const cached = await state.cache.get(file) if (cached) { - const maxBytes = Math.min( - options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES, - DEFAULT_MAX_BASE64_BYTES - ) + const maxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES if (Buffer.byteLength(cached, 'base64') > maxBytes) { return stripBase64(file) } diff --git a/apps/sim/providers/attachments.ts b/apps/sim/providers/attachments.ts index e84daf4b2a5..a429579989b 100644 --- a/apps/sim/providers/attachments.ts +++ b/apps/sim/providers/attachments.ts @@ -110,6 +110,11 @@ export function getProviderAttachmentMaxBytes(_providerId: ProviderId | string): return AGENT_ATTACHMENT_MAX_BYTES } +export function supportsFileAttachments(providerId: ProviderId | string): boolean { + const provider = getAttachmentProvider(providerId) + return Boolean(provider && !UNSUPPORTED_FILE_PROVIDERS.has(provider)) +} + export function inferAttachmentMimeType(file: UserFile): string { const explicitType = file.type?.trim().toLowerCase() if (explicitType && explicitType !== 'application/octet-stream') { From cdd5499832f688860b4fd4eed06311c1539d6bd2 Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan Date: Thu, 14 May 2026 19:14:22 -0700 Subject: [PATCH 5/6] Fix --- apps/sim/providers/anthropic/core.ts | 1 + apps/sim/providers/bedrock/index.ts | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/sim/providers/anthropic/core.ts b/apps/sim/providers/anthropic/core.ts index 5d5951d8ae4..ab080bcceec 100644 --- a/apps/sim/providers/anthropic/core.ts +++ b/apps/sim/providers/anthropic/core.ts @@ -233,6 +233,7 @@ export async function executeAnthropicProviderRequest( const content = buildAnthropicMessageContent(msg.content, msg.files, config.providerId) messages.push({ role: msg.role === 'assistant' ? 'assistant' : 'user', + // double-cast-allowed: shared attachment builder returns Anthropic-compatible content blocks but avoids importing SDK-only union types content: content as unknown as Anthropic.Messages.ContentBlockParam[], }) } diff --git a/apps/sim/providers/bedrock/index.ts b/apps/sim/providers/bedrock/index.ts index d4f4dc22619..22500b84c07 100644 --- a/apps/sim/providers/bedrock/index.ts +++ b/apps/sim/providers/bedrock/index.ts @@ -179,13 +179,11 @@ export const bedrockProvider: ProviderConfig = { } } else { const role: ConversationRole = msg.role === 'assistant' ? 'assistant' : 'user' + const content = buildBedrockMessageContent(msg.content, msg.files, 'bedrock') messages.push({ role, - content: buildBedrockMessageContent( - msg.content, - msg.files, - 'bedrock' - ) as unknown as ContentBlock[], + // double-cast-allowed: shared attachment builder emits Bedrock Converse content blocks while keeping provider-neutral attachment types + content: content as unknown as ContentBlock[], }) } } From 23f1db520993aef9b651941f9a9871e4390baf39 Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan Date: Fri, 15 May 2026 10:10:12 -0700 Subject: [PATCH 6/6] Fix --- apps/sim/providers/attachments.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/sim/providers/attachments.ts b/apps/sim/providers/attachments.ts index a429579989b..d165a1fc9e8 100644 --- a/apps/sim/providers/attachments.ts +++ b/apps/sim/providers/attachments.ts @@ -36,6 +36,19 @@ export interface PreparedProviderAttachment { contentType: 'image' | 'document' | 'audio' | 'video' } +type ProviderMessageInput = { + role: string + content?: string | null + files?: UserFile[] +} + +type ProviderFormattedMessage = { + role: string + content?: string | null | Array> + files?: UserFile[] + [key: string]: unknown +} + const AGENT_ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024 const PDF_MIME_TYPE = 'application/pdf' @@ -532,12 +545,12 @@ export function buildBedrockMessageContent( } export function formatMessagesForProvider( - messages: Array<{ role: string; content?: string | null; files?: UserFile[] }>, + messages: ProviderMessageInput[], providerId: ProviderId | string -) { +): ProviderFormattedMessage[] { return messages.map((message) => { if (!message.files?.length || (message.role !== 'user' && message.role !== 'assistant')) { - return message + return message as ProviderFormattedMessage } const provider = getAttachmentProvider(providerId)