payfrit/payfrit-works
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve menu image analysis - fix JSON error handling

Browse source 
Replaced inline string concatenation with proper struct serialization
for error messages in Claude API calls to avoid JSON escaping issues.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
John Mizerek 2026-01-14 23:08:35 -08:00
parent 307c443f56
commit b069290862
1 changed files with 296 additions and 271 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

567
api/setup/analyzeMenuImages.cfm
View file

@ -2,314 +2,339 @@
<cfsetting enablecfoutputonly="true"> <cfsetting enablecfoutputonly="true">
<cfcontent type="application/json; charset=utf-8" reset="true"> <cfcontent type="application/json; charset=utf-8" reset="true">
<cfscript> <!---
/** Analyze Menu Images using Claude Vision API
* Analyze Menu Images using Claude Vision API
*
* Accepts uploaded menu images (JPG, PNG, PDF) and sends them to Claude
* to extract structured menu data including:
* - Business info (name, address, phone, hours)
* - Categories
* - Modifier templates
* - Menu items with prices and descriptions
*
* Returns structured JSON for the setup wizard to display
*/
response = { "OK": false }; Accepts uploaded menu images (JPG, PNG, PDF) and sends them to Claude
to extract structured menu data including:
- Business info (name, address, phone, hours)
- Categories
- Modifier templates
- Menu items with prices and descriptions
// Claude API key - should be in environment or config Returns structured JSON for the setup wizard to display
CLAUDE_API_KEY = application.claudeApiKey ?: ""; --->
// If not in application scope, try to read from config <cfset response = structNew()>
if (!len(CLAUDE_API_KEY)) { <cfset response["OK"] = false>
try {
// Try relative path from current file location first
configPath = getDirectoryFromPath(getCurrentTemplatePath()) & "../../config/claude.json";
if (fileExists(configPath)) {
configData = deserializeJSON(fileRead(configPath));
CLAUDE_API_KEY = configData.apiKey ?: "";
}
// Fallback to expandPath if relative path didn't work
if (!len(CLAUDE_API_KEY)) {
configPath = expandPath("/biz.payfrit.com/config/claude.json");
if (fileExists(configPath)) {
configData = deserializeJSON(fileRead(configPath));
CLAUDE_API_KEY = configData.apiKey ?: "";
}
}
} catch (any e) {
// Config file doesn't exist or is invalid
}
}
if (!len(CLAUDE_API_KEY)) { <!--- Claude API key - should be in environment or config --->
response.MESSAGE = "Claude API key not configured"; <cfset CLAUDE_API_KEY = "">
writeOutput(serializeJSON(response)); <cfif structKeyExists(application, "claudeApiKey") AND len(application.claudeApiKey)>
abort; <cfset CLAUDE_API_KEY = application.claudeApiKey>
} </cfif>
try { <!--- If not in application scope, try to read from config --->
// Check for uploaded files <cfif NOT len(CLAUDE_API_KEY)>
uploadedFiles = []; <cftry>
<!--- Try relative path from current file location first --->
<cfset configPath = getDirectoryFromPath(getCurrentTemplatePath()) & "../../config/claude.json">
<cfif fileExists(configPath)>
<cfset configData = deserializeJSON(fileRead(configPath))>
<cfif structKeyExists(configData, "apiKey")>
<cfset CLAUDE_API_KEY = configData.apiKey>
</cfif>
</cfif>
<!--- Fallback to expandPath if relative path didn't work --->
<cfif NOT len(CLAUDE_API_KEY)>
<cfset configPath = expandPath("/biz.payfrit.com/config/claude.json")>
<cfif fileExists(configPath)>
<cfset configData = deserializeJSON(fileRead(configPath))>
<cfif structKeyExists(configData, "apiKey")>
<cfset CLAUDE_API_KEY = configData.apiKey>
</cfif>
</cfif>
</cfif>
<cfcatch type="any">
<!--- Config file doesn't exist or is invalid --->
</cfcatch>
</cftry>
</cfif>
// Get form fields for file uploads (file0, file1, file2, etc.) <cfif NOT len(CLAUDE_API_KEY)>
formFields = form.keyArray(); <cfset response["MESSAGE"] = "Claude API key not configured">
for (fieldName in formFields) { <cfoutput>#serializeJSON(response)#</cfoutput>
if (reFindNoCase("^file[0-9]+$", fieldName) && len(form[fieldName])) { <cfabort>
// This is a file field </cfif>
arrayAppend(uploadedFiles, fieldName);
}
}
if (arrayLen(uploadedFiles) == 0) { <cftry>
throw(message="No files uploaded"); <!--- Check for uploaded files --->
} <cfset uploadedFiles = arrayNew(1)>
// Process uploaded files - convert to base64 <!--- Get form fields for file uploads (file0, file1, file2, etc.) --->
imageDataArray = []; <cfset formFields = form.keyArray()>
<cfloop array="#formFields#" index="fieldName">
<cfif reFindNoCase("^file[0-9]+$", fieldName) AND len(form[fieldName])>
<cfset arrayAppend(uploadedFiles, fieldName)>
</cfif>
</cfloop>
// Create temp directory for uploads if needed <cfif arrayLen(uploadedFiles) EQ 0>
uploadDir = getTempDirectory() & "payfrit_menu_uploads/"; <cfthrow message="No files uploaded">
if (!directoryExists(uploadDir)) { </cfif>
directoryCreate(uploadDir);
}
for (fieldName in uploadedFiles) { <!--- Process uploaded files - convert to base64 --->
try { <cfset imageDataArray = arrayNew(1)>
// Upload the file
uploadResult = fileUpload(
uploadDir,
fieldName,
"image/jpeg,image/png,image/gif,image/webp,application/pdf",
"makeunique"
);
} catch (any uploadErr) {
throw(message="File upload failed: " & uploadErr.message, detail="Field: " & fieldName);
}
if (uploadResult.fileWasSaved) { <!--- Create temp directory for uploads if needed --->
filePath = uploadResult.serverDirectory & "/" & uploadResult.serverFile; <cfset uploadDir = getTempDirectory() & "payfrit_menu_uploads/">
fileExt = lCase(uploadResult.serverFileExt); <cfif NOT directoryExists(uploadDir)>
<cfdirectory action="create" directory="#uploadDir#">
</cfif>
// For images, resize if too large (max 1600px on longest side) <cfloop array="#uploadedFiles#" index="fieldName">
if (listFindNoCase("jpg,jpeg,png,gif,webp", fileExt)) { <!--- Upload the file --->
try { <cffile action="upload" destination="#uploadDir#" filefield="#fieldName#"
img = imageRead(filePath); accept="image/jpeg,image/png,image/gif,image/webp,application/pdf"
} catch (any readErr) { nameconflict="makeunique" result="uploadResult">
throw(message="Image read failed: " & readErr.message, detail="File: " & filePath);
}
imgWidth = img.width; <cfif uploadResult.fileWasSaved>
imgHeight = img.height; <cfset filePath = uploadResult.serverDirectory & "/" & uploadResult.serverFile>
maxDimension = 1600; <cfset fileExt = lCase(uploadResult.serverFileExt)>
if (imgWidth > maxDimension || imgHeight > maxDimension) { <!--- For images, resize if too large (max 1600px on longest side) --->
try { <cfif listFindNoCase("jpg,jpeg,png,gif,webp", fileExt)>
if (imgWidth > imgHeight) { <cfimage action="read" source="#filePath#" name="img">
newWidth = maxDimension; <cfset imgWidth = img.width>
newHeight = int(imgHeight * (maxDimension / imgWidth)); <cfset imgHeight = img.height>
} else { <cfset maxDimension = 1600>
newHeight = maxDimension;
newWidth = int(imgWidth * (maxDimension / imgHeight));
}
imageResize(img, newWidth, newHeight);
} catch (any resizeErr) {
throw(message="Image resize failed: " & resizeErr.message, detail="Dimensions: " & imgWidth & "x" & imgHeight);
}
}
// Re-save with good quality compression
try {
imageWrite(img, filePath, 0.8);
} catch (any writeErr) {
throw(message="Image write failed: " & writeErr.message, detail="File: " & filePath);
}
}
// Read file and convert to base64 <cfif imgWidth GT maxDimension OR imgHeight GT maxDimension>
fileContent = fileReadBinary(filePath); <cfif imgWidth GT imgHeight>
base64Content = toBase64(fileContent); <cfset newWidth = maxDimension>
<cfset newHeight = int(imgHeight * (maxDimension / imgWidth))>
<cfelse>
<cfset newHeight = maxDimension>
<cfset newWidth = int(imgWidth * (maxDimension / imgHeight))>
</cfif>
<cfimage action="resize" source="#img#" width="#newWidth#" height="#newHeight#" name="img">
</cfif>
<!--- Re-save with good quality compression --->
<cfimage action="write" source="#img#" destination="#filePath#" quality="0.8" overwrite="true">
</cfif>
// Determine media type <!--- Read file and convert to base64 --->
mediaType = "image/jpeg"; <cffile action="readbinary" file="#filePath#" variable="fileContent">
if (fileExt == "png") mediaType = "image/png"; <cfset base64Content = toBase64(fileContent)>
else if (fileExt == "gif") mediaType = "image/gif";
else if (fileExt == "webp") mediaType = "image/webp";
else if (fileExt == "pdf") mediaType = "application/pdf";
// Claude API uses different structure for PDFs vs images <!--- Determine media type --->
if (fileExt == "pdf") { <cfset mediaType = "image/jpeg">
arrayAppend(imageDataArray, { <cfif fileExt EQ "png">
"type": "document", <cfset mediaType = "image/png">
"source": { <cfelseif fileExt EQ "gif">
"type": "base64", <cfset mediaType = "image/gif">
"media_type": "application/pdf", <cfelseif fileExt EQ "webp">
"data": base64Content <cfset mediaType = "image/webp">
} <cfelseif fileExt EQ "pdf">
}); <cfset mediaType = "application/pdf">
} else { </cfif>
arrayAppend(imageDataArray, {
"type": "image",
"source": {
"type": "base64",
"media_type": mediaType,
"data": base64Content
}
});
}
// Clean up temp file <!--- Claude API uses different structure for PDFs vs images --->
fileDelete(filePath); <cfif fileExt EQ "pdf">
} <cfset imgStruct = structNew()>
} <cfset imgStruct["type"] = "document">
<cfset imgStruct["source"] = structNew()>
<cfset imgStruct["source"]["type"] = "base64">
<cfset imgStruct["source"]["media_type"] = "application/pdf">
<cfset imgStruct["source"]["data"] = base64Content>
<cfset arrayAppend(imageDataArray, imgStruct)>
<cfelse>
<cfset imgStruct = structNew()>
<cfset imgStruct["type"] = "image">
<cfset imgStruct["source"] = structNew()>
<cfset imgStruct["source"]["type"] = "base64">
<cfset imgStruct["source"]["media_type"] = mediaType>
<cfset imgStruct["source"]["data"] = base64Content>
<cfset arrayAppend(imageDataArray, imgStruct)>
</cfif>
if (arrayLen(imageDataArray) == 0) { <!--- Clean up temp file --->
throw(message="No valid images could be processed"); <cffile action="delete" file="#filePath#">
} </cfif>
</cfloop>
// Build the prompt for Claude <cfif arrayLen(imageDataArray) EQ 0>
systemPrompt = "You are an expert at extracting structured menu data from restaurant menu images. Your task is to analyze menu images and return the data in a specific JSON format. Be thorough and extract ALL items, categories, and any modifier patterns you can identify."; <cfthrow message="No valid images could be processed">
</cfif>
userPrompt = "Please analyze these menu images and extract all the information into a JSON object with these keys: business (with name, address, phone, hours), categories (array with name and itemCount), modifiers (array with name, required boolean, and options array), and items (array with name, description, price, category, and modifiers array). Extract EVERY menu item with accurate prices. Group items by category. Look for modifier patterns like sizes, protein choices, bread choices, and add-ons. Create reusable modifier templates for patterns that appear on multiple items. Return ONLY valid JSON, no other text."; <!--- Build the prompt for Claude --->
<cfset systemPrompt = "You are an expert at extracting structured menu data from restaurant menu images. Your task is to analyze menu images and return the data in a specific JSON format. Be thorough and extract ALL items, categories, and any modifier patterns you can identify.">
// Build the messages array with images <cfset userPrompt = "Please analyze these menu images and extract all the information into a JSON object with these keys: business (with name, address, phone, hours), categories (array with name and itemCount), modifiers (array with name, required boolean, and options array), and items (array with name, description, price, category, and modifiers array). Extract EVERY menu item with accurate prices. Group items by category. Look for modifier patterns like sizes, protein choices, bread choices, and add-ons. Create reusable modifier templates for patterns that appear on multiple items. Return ONLY valid JSON, no other text.">
messagesContent = [];
// Add each image <!--- Build the messages array with images --->
for (imgData in imageDataArray) { <cfset messagesContent = arrayNew(1)>
arrayAppend(messagesContent, imgData);
}
// Add the text prompt <!--- Add each image --->
arrayAppend(messagesContent, { <cfloop array="#imageDataArray#" index="imgData">
"type": "text", <cfset arrayAppend(messagesContent, imgData)>
"text": userPrompt </cfloop>
});
// Build request body <!--- Add the text prompt --->
requestBody = { <cfset textBlock = structNew()>
"model": "claude-sonnet-4-20250514", <cfset textBlock["type"] = "text">
"max_tokens": 8192, <cfset textBlock["text"] = userPrompt>
"system": systemPrompt, <cfset arrayAppend(messagesContent, textBlock)>
"messages": [
{
"role": "user",
"content": messagesContent
}
]
};
// Write request body to temp file (Lucee's HTTP client has issues with large payloads) <!--- Build request body --->
tempRequestFile = getTempDirectory() & "claude_request_" & createUUID() & ".json"; <cfset userMessage = structNew()>
tempResponseFile = getTempDirectory() & "claude_response_" & createUUID() & ".json"; <cfset userMessage["role"] = "user">
fileWrite(tempRequestFile, serializeJSON(requestBody)); <cfset userMessage["content"] = messagesContent>
try { <cfset requestBody = structNew()>
// Call Claude API using cfhttp tag (more reliable than http component) <cfset requestBody["model"] = "claude-sonnet-4-20250514">
httpResult = {}; <cfset requestBody["max_tokens"] = 8192>
http url="https://api.anthropic.com/v1/messages" method="POST" timeout=300 result="httpResult" { <cfset requestBody["system"] = systemPrompt>
httpparam type="header" name="Content-Type" value="application/json"; <cfset requestBody["messages"] = arrayNew(1)>
httpparam type="header" name="x-api-key" value=CLAUDE_API_KEY; <cfset arrayAppend(requestBody["messages"], userMessage)>
httpparam type="header" name="anthropic-version" value="2023-06-01";
httpparam type="body" value=serializeJSON(requestBody);
}
httpStatusCode = httpResult.statusCode ?: "0"; <!--- Call Claude API using Java HttpURLConnection (bypasses Lucee's HTTP client issues) --->
httpFileContent = httpResult.fileContent ?: ""; <cfset requestJson = serializeJSON(requestBody)>
<cfset httpFileContent = "">
<cfset httpStatusCode = 0>
// Normalize status code <cftry>
if (isNumeric(httpStatusCode)) { <cfset urlObj = createObject("java", "java.net.URL").init("https://api.anthropic.com/v1/messages")>
httpStatusCode = int(httpStatusCode); <cfset conn = urlObj.openConnection()>
} else if (findNoCase("200", httpStatusCode)) { <cfset conn.setRequestMethod("POST")>
httpStatusCode = 200; <cfset conn.setDoOutput(true)>
} <cfset conn.setConnectTimeout(30000)>
<cfset conn.setReadTimeout(300000)>
<cfset conn.setRequestProperty("Content-Type", "application/json")>
<cfset conn.setRequestProperty("x-api-key", CLAUDE_API_KEY)>
<cfset conn.setRequestProperty("anthropic-version", "2023-06-01")>
if (httpStatusCode != 200) { <!--- Write request body --->
// Try to parse error message from Claude's response <cfset outputStream = conn.getOutputStream()>
errorMsg = "Claude API error: " & httpResult.statusCode; <cfset writer = createObject("java", "java.io.OutputStreamWriter").init(outputStream, "UTF-8")>
try { <cfset writer.write(requestJson)>
errorData = deserializeJSON(httpFileContent); <cfset writer.flush()>
if (structKeyExists(errorData, "error") && structKeyExists(errorData.error, "message")) { <cfset writer.close()>
errorMsg = errorMsg & " - " & errorData.error.message;
}
} catch (any e) {
// Use raw response if can't parse
}
throw(message=errorMsg, detail=httpFileContent);
}
} finally {
// Clean up temp files
if (fileExists(tempRequestFile)) fileDelete(tempRequestFile);
}
// Parse Claude's response <!--- Get response --->
claudeResponse = deserializeJSON(httpFileContent); <cfset httpStatusCode = conn.getResponseCode()>
if (!structKeyExists(claudeResponse, "content") || !arrayLen(claudeResponse.content)) { <!--- Read response body --->
throw(message="Empty response from Claude"); <cfif httpStatusCode GTE 200 AND httpStatusCode LT 300>
} <cfset inputStream = conn.getInputStream()>
<cfelse>
<cfset inputStream = conn.getErrorStream()>
</cfif>
// Extract the text content <cfif NOT isNull(inputStream)>
responseText = ""; <cfset reader = createObject("java", "java.io.BufferedReader").init(
for (block in claudeResponse.content) { createObject("java", "java.io.InputStreamReader").init(inputStream, "UTF-8")
if (block.type == "text") { )>
responseText = block.text; <cfset sb = createObject("java", "java.lang.StringBuilder").init()>
break; <cfset line = reader.readLine()>
} <cfloop condition="NOT isNull(line)">
} <cfset sb.append(line)>
<cfset line = reader.readLine()>
</cfloop>
<cfset reader.close()>
<cfset httpFileContent = sb.toString()>
</cfif>
// Parse the JSON from Claude's response <cfset conn.disconnect()>
// Claude might wrap it in markdown code blocks, so strip those
responseText = trim(responseText);
if (left(responseText, 7) == "```json") {
responseText = mid(responseText, 8, len(responseText) - 7);
}
if (left(responseText, 3) == "```") {
responseText = mid(responseText, 4, len(responseText) - 3);
}
if (right(responseText, 3) == "```") {
responseText = left(responseText, len(responseText) - 3);
}
responseText = trim(responseText);
extractedData = deserializeJSON(responseText); <cfcatch type="any">
<cfset httpStatusCode = 0>
<cfset errorStruct = structNew()>
<cfset errorStruct["error"] = structNew()>
<cfset errorStruct["error"]["message"] = cfcatch.message>
<cfset httpFileContent = serializeJSON(errorStruct)>
</cfcatch>
</cftry>
// Update category item counts <!--- Normalize status code --->
if (structKeyExists(extractedData, "categories") && structKeyExists(extractedData, "items")) { <cfif isNumeric(httpStatusCode)>
for (i = 1; i <= arrayLen(extractedData.categories); i++) { <cfset httpStatusCode = int(httpStatusCode)>
catName = extractedData.categories[i].name; <cfelseif findNoCase("200", httpStatusCode)>
itemCount = 0; <cfset httpStatusCode = 200>
for (item in extractedData.items) { </cfif>
if (structKeyExists(item, "category") && item.category == catName) {
itemCount++;
}
}
extractedData.categories[i].itemCount = itemCount;
}
}
// Add unique IDs to items <cfif httpStatusCode NEQ 200>
if (structKeyExists(extractedData, "items")) { <!--- Try to parse error message from Claude's response --->
for (i = 1; i <= arrayLen(extractedData.items); i++) { <cfset errorMsg = "Claude API error: " & httpStatusCode>
extractedData.items[i].id = "item_" & i; <cftry>
} <cfset errorData = deserializeJSON(httpFileContent)>
} <cfif structKeyExists(errorData, "error") AND structKeyExists(errorData.error, "message")>
<cfset errorMsg = errorMsg & " - " & errorData.error.message>
</cfif>
<cfcatch type="any">
<!--- Use raw response if can't parse --->
</cfcatch>
</cftry>
<cfthrow message="#errorMsg#" detail="#httpFileContent#">
</cfif>
response.OK = true; <!--- Parse Claude's response --->
response.DATA = extractedData; <cfset claudeResponse = deserializeJSON(httpFileContent)>
response.imagesProcessed = arrayLen(imageDataArray);
} catch (any e) { <cfif NOT structKeyExists(claudeResponse, "content") OR NOT arrayLen(claudeResponse.content)>
response.MESSAGE = e.message; <cfthrow message="Empty response from Claude">
if (len(e.detail)) { </cfif>
response.DETAIL = e.detail;
}
// Add debug info
if (structKeyExists(e, "tagContext") && arrayLen(e.tagContext) > 0) {
response.DEBUG_LINE = e.tagContext[1].line;
response.DEBUG_TEMPLATE = e.tagContext[1].template;
}
}
writeOutput(serializeJSON(response)); <!--- Extract the text content --->
</cfscript> <cfset responseText = "">
<cfloop array="#claudeResponse.content#" index="block">
<cfif block.type EQ "text">
<cfset responseText = block.text>
<cfbreak>
</cfif>
</cfloop>
<!--- Parse the JSON from Claude's response --->
<!--- Claude might wrap it in markdown code blocks, so strip those --->
<cfset responseText = trim(responseText)>
<cfif left(responseText, 7) EQ "```json">
<cfset responseText = mid(responseText, 8, len(responseText) - 7)>
</cfif>
<cfif left(responseText, 3) EQ "```">
<cfset responseText = mid(responseText, 4, len(responseText) - 3)>
</cfif>
<cfif right(responseText, 3) EQ "```">
<cfset responseText = left(responseText, len(responseText) - 3)>
</cfif>
<cfset responseText = trim(responseText)>
<cfset extractedData = deserializeJSON(responseText)>
<!--- Update category item counts --->
<cfif structKeyExists(extractedData, "categories") AND structKeyExists(extractedData, "items")>
<cfloop from="1" to="#arrayLen(extractedData.categories)#" index="i">
<cfset catName = extractedData.categories[i].name>
<cfset itemCount = 0>
<cfloop array="#extractedData.items#" index="item">
<cfif structKeyExists(item, "category") AND item.category EQ catName>
<cfset itemCount = itemCount + 1>
</cfif>
</cfloop>
<cfset extractedData.categories[i]["itemCount"] = itemCount>
</cfloop>
</cfif>
<!--- Add unique IDs to items --->
<cfif structKeyExists(extractedData, "items")>
<cfloop from="1" to="#arrayLen(extractedData.items)#" index="i">
<cfset extractedData.items[i]["id"] = "item_" & i>
</cfloop>
</cfif>
<cfset response["OK"] = true>
<cfset response["DATA"] = extractedData>
<cfset response["imagesProcessed"] = arrayLen(imageDataArray)>
<cfcatch type="any">
<cfset response["MESSAGE"] = cfcatch.message>
<cfif len(cfcatch.detail)>
<cfset response["DETAIL"] = cfcatch.detail>
</cfif>
<cfif structKeyExists(cfcatch, "tagContext") AND arrayLen(cfcatch.tagContext) GT 0>
<cfset response["DEBUG_LINE"] = cfcatch.tagContext[1].line>
<cfset response["DEBUG_TEMPLATE"] = cfcatch.tagContext[1].template>
</cfif>
</cfcatch>
</cftry>
<cfoutput>#serializeJSON(response)#</cfoutput>