/** * Analyze Menu Images using Claude Vision API * * Accepts uploaded menu images (JPG, PNG, PDF) and sends them to Claude * to extract structured menu data including: * - Business info (name, address, phone, hours) * - Categories * - Modifier templates * - Menu items with prices and descriptions * * Returns structured JSON for the setup wizard to display */ response = { "OK": false }; // Claude API key - should be in environment or config CLAUDE_API_KEY = application.claudeApiKey ?: ""; // If not in application scope, try to read from config if (!len(CLAUDE_API_KEY)) { try { configPath = expandPath("/biz.payfrit.com/config/claude.json"); if (fileExists(configPath)) { configData = deserializeJSON(fileRead(configPath)); CLAUDE_API_KEY = configData.apiKey ?: ""; } } catch (any e) { // Config file doesn't exist or is invalid } } if (!len(CLAUDE_API_KEY)) { response.MESSAGE = "Claude API key not configured"; writeOutput(serializeJSON(response)); abort; } try { // Check for uploaded files uploadedFiles = []; // Get form fields for file uploads (file0, file1, file2, etc.) formFields = form.keyArray(); for (fieldName in formFields) { if (reFindNoCase("^file\d+$", fieldName) && len(form[fieldName])) { // This is a file field arrayAppend(uploadedFiles, fieldName); } } if (arrayLen(uploadedFiles) == 0) { throw(message="No files uploaded"); } // Process uploaded files - convert to base64 imageDataArray = []; // Create temp directory for uploads if needed uploadDir = getTempDirectory() & "payfrit_menu_uploads/"; if (!directoryExists(uploadDir)) { directoryCreate(uploadDir); } for (fieldName in uploadedFiles) { // Upload the file uploadResult = fileUpload( uploadDir, fieldName, "image/jpeg,image/png,image/gif,image/webp,application/pdf", "makeunique" ); if (uploadResult.fileWasSaved) { filePath = uploadResult.serverDirectory & "/" & uploadResult.serverFile; fileExt = lCase(uploadResult.serverFileExt); // Read file and convert to base64 fileContent = fileReadBinary(filePath); base64Content = toBase64(fileContent); // Determine media type mediaType = "image/jpeg"; if (fileExt == "png") mediaType = "image/png"; else if (fileExt == "gif") mediaType = "image/gif"; else if (fileExt == "webp") mediaType = "image/webp"; else if (fileExt == "pdf") mediaType = "application/pdf"; arrayAppend(imageDataArray, { "type": "image", "source": { "type": "base64", "media_type": mediaType, "data": base64Content } }); // Clean up temp file fileDelete(filePath); } } if (arrayLen(imageDataArray) == 0) { throw(message="No valid images could be processed"); } // Build the prompt for Claude systemPrompt = "You are an expert at extracting structured menu data from restaurant menu images. Your task is to analyze menu images and return the data in a specific JSON format. Be thorough and extract ALL items, categories, and any modifier patterns you can identify."; userPrompt = "Please analyze these menu images and extract all the information into the following JSON structure: { ""business"": { ""name"": ""Restaurant name if visible"", ""address"": ""Full address if visible"", ""phone"": ""Phone number(s) if visible"", ""hours"": ""Business hours if visible"" }, ""categories"": [ { ""name"": ""Category name"", ""itemCount"": 0 } ], ""modifiers"": [ { ""name"": ""Modifier template name (e.g., 'Size', 'Protein Choice', 'Toast Type')"", ""required"": true or false, ""options"": [ { ""name"": ""Option name"", ""price"": 0.00 } ] } ], ""items"": [ { ""name"": ""Item name"", ""description"": ""Item description if any"", ""price"": 0.00, ""category"": ""Category this item belongs to"", ""modifiers"": [""Name of applicable modifier template""] } ] } Important guidelines: 1. Extract EVERY menu item you can see, with accurate prices 2. Group items into their categories as shown on the menu 3. Look for modifier patterns like: - Size options (Small/Regular/Large) - Protein choices (Chicken/Beef/Shrimp) - Bread choices (Wheat/Sourdough/White/Rye) - Add-ons with prices (Add bacon +$2.00) - Required choices (served with choice of side) 4. For modifiers that appear on multiple items, create a reusable template 5. Assign the appropriate modifier templates to each item 6. If a price has a '+' it's usually an upcharge modifier option 7. Pay attention to category headers and section dividers Return ONLY the JSON object, no other text."; // Build the messages array with images messagesContent = []; // Add each image for (imgData in imageDataArray) { arrayAppend(messagesContent, imgData); } // Add the text prompt arrayAppend(messagesContent, { "type": "text", "text": userPrompt }); // Call Claude API httpService = new http(); httpService.setMethod("POST"); httpService.setUrl("https://api.anthropic.com/v1/messages"); httpService.addParam(type="header", name="Content-Type", value="application/json"); httpService.addParam(type="header", name="x-api-key", value=CLAUDE_API_KEY); httpService.addParam(type="header", name="anthropic-version", value="2023-06-01"); requestBody = { "model": "claude-sonnet-4-20250514", "max_tokens": 8192, "system": systemPrompt, "messages": [ { "role": "user", "content": messagesContent } ] }; httpService.addParam(type="body", value=serializeJSON(requestBody)); httpResult = httpService.send().getPrefix(); if (httpResult.statusCode != "200 OK" && httpResult.statusCode != 200) { throw(message="Claude API error: " & httpResult.statusCode, detail=httpResult.fileContent); } // Parse Claude's response claudeResponse = deserializeJSON(httpResult.fileContent); if (!structKeyExists(claudeResponse, "content") || !arrayLen(claudeResponse.content)) { throw(message="Empty response from Claude"); } // Extract the text content responseText = ""; for (block in claudeResponse.content) { if (block.type == "text") { responseText = block.text; break; } } // Parse the JSON from Claude's response // Claude might wrap it in markdown code blocks, so strip those responseText = trim(responseText); if (left(responseText, 7) == "```json") { responseText = mid(responseText, 8, len(responseText) - 7); } if (left(responseText, 3) == "```") { responseText = mid(responseText, 4, len(responseText) - 3); } if (right(responseText, 3) == "```") { responseText = left(responseText, len(responseText) - 3); } responseText = trim(responseText); extractedData = deserializeJSON(responseText); // Update category item counts if (structKeyExists(extractedData, "categories") && structKeyExists(extractedData, "items")) { for (i = 1; i <= arrayLen(extractedData.categories); i++) { catName = extractedData.categories[i].name; itemCount = 0; for (item in extractedData.items) { if (structKeyExists(item, "category") && item.category == catName) { itemCount++; } } extractedData.categories[i].itemCount = itemCount; } } // Add unique IDs to items if (structKeyExists(extractedData, "items")) { for (i = 1; i <= arrayLen(extractedData.items); i++) { extractedData.items[i].id = "item_" & i; } } response.OK = true; response.DATA = extractedData; response.imagesProcessed = arrayLen(imageDataArray); } catch (any e) { response.MESSAGE = e.message; if (len(e.detail)) { response.DETAIL = e.detail; } } writeOutput(serializeJSON(response));