/** * Analyze Menu Images using Claude Vision API * * Accepts uploaded menu images (JPG, PNG, PDF) and sends them to Claude * to extract structured menu data including: * - Business info (name, address, phone, hours) * - Categories * - Modifier templates * - Menu items with prices and descriptions * * Returns structured JSON for the setup wizard to display */ response = { "OK": false }; // Claude API key - should be in environment or config CLAUDE_API_KEY = application.claudeApiKey ?: ""; // If not in application scope, try to read from config if (!len(CLAUDE_API_KEY)) { try { // Try relative path from current file location first configPath = getDirectoryFromPath(getCurrentTemplatePath()) & "../../config/claude.json"; if (fileExists(configPath)) { configData = deserializeJSON(fileRead(configPath)); CLAUDE_API_KEY = configData.apiKey ?: ""; } // Fallback to expandPath if relative path didn't work if (!len(CLAUDE_API_KEY)) { configPath = expandPath("/biz.payfrit.com/config/claude.json"); if (fileExists(configPath)) { configData = deserializeJSON(fileRead(configPath)); CLAUDE_API_KEY = configData.apiKey ?: ""; } } } catch (any e) { // Config file doesn't exist or is invalid } } if (!len(CLAUDE_API_KEY)) { response.MESSAGE = "Claude API key not configured"; writeOutput(serializeJSON(response)); abort; } try { // Check for uploaded files uploadedFiles = []; // Get form fields for file uploads (file0, file1, file2, etc.) formFields = form.keyArray(); for (fieldName in formFields) { if (reFindNoCase("^file[0-9]+$", fieldName) && len(form[fieldName])) { // This is a file field arrayAppend(uploadedFiles, fieldName); } } if (arrayLen(uploadedFiles) == 0) { throw(message="No files uploaded"); } // Process uploaded files - convert to base64 imageDataArray = []; // Create temp directory for uploads if needed uploadDir = getTempDirectory() & "payfrit_menu_uploads/"; if (!directoryExists(uploadDir)) { directoryCreate(uploadDir); } for (fieldName in uploadedFiles) { try { // Upload the file uploadResult = fileUpload( uploadDir, fieldName, "image/jpeg,image/png,image/gif,image/webp,application/pdf", "makeunique" ); } catch (any uploadErr) { throw(message="File upload failed: " & uploadErr.message, detail="Field: " & fieldName); } if (uploadResult.fileWasSaved) { filePath = uploadResult.serverDirectory & "/" & uploadResult.serverFile; fileExt = lCase(uploadResult.serverFileExt); // For images, resize if too large (max 1600px on longest side) if (listFindNoCase("jpg,jpeg,png,gif,webp", fileExt)) { try { img = imageRead(filePath); } catch (any readErr) { throw(message="Image read failed: " & readErr.message, detail="File: " & filePath); } imgWidth = img.width; imgHeight = img.height; maxDimension = 1600; if (imgWidth > maxDimension || imgHeight > maxDimension) { try { if (imgWidth > imgHeight) { newWidth = maxDimension; newHeight = int(imgHeight * (maxDimension / imgWidth)); } else { newHeight = maxDimension; newWidth = int(imgWidth * (maxDimension / imgHeight)); } imageResize(img, newWidth, newHeight); } catch (any resizeErr) { throw(message="Image resize failed: " & resizeErr.message, detail="Dimensions: " & imgWidth & "x" & imgHeight); } } // Re-save with good quality compression try { imageWrite(img, filePath, 0.8); } catch (any writeErr) { throw(message="Image write failed: " & writeErr.message, detail="File: " & filePath); } } // Read file and convert to base64 fileContent = fileReadBinary(filePath); base64Content = toBase64(fileContent); // Determine media type mediaType = "image/jpeg"; if (fileExt == "png") mediaType = "image/png"; else if (fileExt == "gif") mediaType = "image/gif"; else if (fileExt == "webp") mediaType = "image/webp"; else if (fileExt == "pdf") mediaType = "application/pdf"; // Claude API uses different structure for PDFs vs images if (fileExt == "pdf") { arrayAppend(imageDataArray, { "type": "document", "source": { "type": "base64", "media_type": "application/pdf", "data": base64Content } }); } else { arrayAppend(imageDataArray, { "type": "image", "source": { "type": "base64", "media_type": mediaType, "data": base64Content } }); } // Clean up temp file fileDelete(filePath); } } if (arrayLen(imageDataArray) == 0) { throw(message="No valid images could be processed"); } // Build the prompt for Claude systemPrompt = "You are an expert at extracting structured menu data from restaurant menu images. Your task is to analyze menu images and return the data in a specific JSON format. Be thorough and extract ALL items, categories, and any modifier patterns you can identify."; userPrompt = "Please analyze these menu images and extract all the information into a JSON object with these keys: business (with name, address, phone, hours), categories (array with name and itemCount), modifiers (array with name, required boolean, and options array), and items (array with name, description, price, category, and modifiers array). Extract EVERY menu item with accurate prices. Group items by category. Look for modifier patterns like sizes, protein choices, bread choices, and add-ons. Create reusable modifier templates for patterns that appear on multiple items. Return ONLY valid JSON, no other text."; // Build the messages array with images messagesContent = []; // Add each image for (imgData in imageDataArray) { arrayAppend(messagesContent, imgData); } // Add the text prompt arrayAppend(messagesContent, { "type": "text", "text": userPrompt }); // Build request body requestBody = { "model": "claude-sonnet-4-20250514", "max_tokens": 8192, "system": systemPrompt, "messages": [ { "role": "user", "content": messagesContent } ] }; // Write request body to temp file (Lucee's HTTP client has issues with large payloads) tempRequestFile = getTempDirectory() & "claude_request_" & createUUID() & ".json"; tempResponseFile = getTempDirectory() & "claude_response_" & createUUID() & ".json"; fileWrite(tempRequestFile, serializeJSON(requestBody)); try { // Call Claude API using cfhttp tag (more reliable than http component) httpResult = {}; http url="https://api.anthropic.com/v1/messages" method="POST" timeout=300 result="httpResult" { httpparam type="header" name="Content-Type" value="application/json"; httpparam type="header" name="x-api-key" value=CLAUDE_API_KEY; httpparam type="header" name="anthropic-version" value="2023-06-01"; httpparam type="body" value=serializeJSON(requestBody); } httpStatusCode = httpResult.statusCode ?: "0"; httpFileContent = httpResult.fileContent ?: ""; // Normalize status code if (isNumeric(httpStatusCode)) { httpStatusCode = int(httpStatusCode); } else if (findNoCase("200", httpStatusCode)) { httpStatusCode = 200; } if (httpStatusCode != 200) { // Try to parse error message from Claude's response errorMsg = "Claude API error: " & httpResult.statusCode; try { errorData = deserializeJSON(httpFileContent); if (structKeyExists(errorData, "error") && structKeyExists(errorData.error, "message")) { errorMsg = errorMsg & " - " & errorData.error.message; } } catch (any e) { // Use raw response if can't parse } throw(message=errorMsg, detail=httpFileContent); } } finally { // Clean up temp files if (fileExists(tempRequestFile)) fileDelete(tempRequestFile); } // Parse Claude's response claudeResponse = deserializeJSON(httpFileContent); if (!structKeyExists(claudeResponse, "content") || !arrayLen(claudeResponse.content)) { throw(message="Empty response from Claude"); } // Extract the text content responseText = ""; for (block in claudeResponse.content) { if (block.type == "text") { responseText = block.text; break; } } // Parse the JSON from Claude's response // Claude might wrap it in markdown code blocks, so strip those responseText = trim(responseText); if (left(responseText, 7) == "```json") { responseText = mid(responseText, 8, len(responseText) - 7); } if (left(responseText, 3) == "```") { responseText = mid(responseText, 4, len(responseText) - 3); } if (right(responseText, 3) == "```") { responseText = left(responseText, len(responseText) - 3); } responseText = trim(responseText); extractedData = deserializeJSON(responseText); // Update category item counts if (structKeyExists(extractedData, "categories") && structKeyExists(extractedData, "items")) { for (i = 1; i <= arrayLen(extractedData.categories); i++) { catName = extractedData.categories[i].name; itemCount = 0; for (item in extractedData.items) { if (structKeyExists(item, "category") && item.category == catName) { itemCount++; } } extractedData.categories[i].itemCount = itemCount; } } // Add unique IDs to items if (structKeyExists(extractedData, "items")) { for (i = 1; i <= arrayLen(extractedData.items); i++) { extractedData.items[i].id = "item_" & i; } } response.OK = true; response.DATA = extractedData; response.imagesProcessed = arrayLen(imageDataArray); } catch (any e) { response.MESSAGE = e.message; if (len(e.detail)) { response.DETAIL = e.detail; } // Add debug info if (structKeyExists(e, "tagContext") && arrayLen(e.tagContext) > 0) { response.DEBUG_LINE = e.tagContext[1].line; response.DEBUG_TEMPLATE = e.tagContext[1].template; } } writeOutput(serializeJSON(response));