payfrit/payfrit-works
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve menu image analysis - fix JSON error handling

Browse source 
Replaced inline string concatenation with proper struct serialization
for error messages in Claude API calls to avoid JSON escaping issues.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
John Mizerek 2026-01-14 23:08:35 -08:00
parent 307c443f56
commit b069290862
1 changed files with 296 additions and 271 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

567
api/setup/analyzeMenuImages.cfm
View file

@ -2,314 +2,339 @@
<cfsetting enablecfoutputonly="true">
<cfcontent type="application/json; charset=utf-8" reset="true">
<cfscript>
/**
* Analyze Menu Images using Claude Vision API
*
* Accepts uploaded menu images (JPG, PNG, PDF) and sends them to Claude
* to extract structured menu data including:
* - Business info (name, address, phone, hours)
* - Categories
* - Modifier templates
* - Menu items with prices and descriptions
*
* Returns structured JSON for the setup wizard to display
*/
<!---
Analyze Menu Images using Claude Vision API
response = { "OK": false };
Accepts uploaded menu images (JPG, PNG, PDF) and sends them to Claude
to extract structured menu data including:
- Business info (name, address, phone, hours)
- Categories
- Modifier templates
- Menu items with prices and descriptions
// Claude API key - should be in environment or config
CLAUDE_API_KEY = application.claudeApiKey ?: "";
Returns structured JSON for the setup wizard to display
--->
// If not in application scope, try to read from config
if (!len(CLAUDE_API_KEY)) {
try {
// Try relative path from current file location first
configPath = getDirectoryFromPath(getCurrentTemplatePath()) & "../../config/claude.json";
if (fileExists(configPath)) {
configData = deserializeJSON(fileRead(configPath));
CLAUDE_API_KEY = configData.apiKey ?: "";
}
// Fallback to expandPath if relative path didn't work
if (!len(CLAUDE_API_KEY)) {
configPath = expandPath("/biz.payfrit.com/config/claude.json");
if (fileExists(configPath)) {
configData = deserializeJSON(fileRead(configPath));
CLAUDE_API_KEY = configData.apiKey ?: "";
}
}
} catch (any e) {
// Config file doesn't exist or is invalid
}
}
<cfset response = structNew()>
<cfset response["OK"] = false>
if (!len(CLAUDE_API_KEY)) {
response.MESSAGE = "Claude API key not configured";
writeOutput(serializeJSON(response));
abort;
}
<!--- Claude API key - should be in environment or config --->
<cfset CLAUDE_API_KEY = "">
<cfif structKeyExists(application, "claudeApiKey") AND len(application.claudeApiKey)>
<cfset CLAUDE_API_KEY = application.claudeApiKey>
</cfif>
try {
// Check for uploaded files
uploadedFiles = [];
<!--- If not in application scope, try to read from config --->
<cfif NOT len(CLAUDE_API_KEY)>
<cftry>
<!--- Try relative path from current file location first --->
<cfset configPath = getDirectoryFromPath(getCurrentTemplatePath()) & "../../config/claude.json">
<cfif fileExists(configPath)>
<cfset configData = deserializeJSON(fileRead(configPath))>
<cfif structKeyExists(configData, "apiKey")>
<cfset CLAUDE_API_KEY = configData.apiKey>
</cfif>
</cfif>
<!--- Fallback to expandPath if relative path didn't work --->
<cfif NOT len(CLAUDE_API_KEY)>
<cfset configPath = expandPath("/biz.payfrit.com/config/claude.json")>
<cfif fileExists(configPath)>
<cfset configData = deserializeJSON(fileRead(configPath))>
<cfif structKeyExists(configData, "apiKey")>
<cfset CLAUDE_API_KEY = configData.apiKey>
</cfif>
</cfif>
</cfif>
<cfcatch type="any">
<!--- Config file doesn't exist or is invalid --->
</cfcatch>
</cftry>
</cfif>
// Get form fields for file uploads (file0, file1, file2, etc.)
formFields = form.keyArray();
for (fieldName in formFields) {
if (reFindNoCase("^file[0-9]+$", fieldName) && len(form[fieldName])) {
// This is a file field
arrayAppend(uploadedFiles, fieldName);
}
}
<cfif NOT len(CLAUDE_API_KEY)>
<cfset response["MESSAGE"] = "Claude API key not configured">
<cfoutput>#serializeJSON(response)#</cfoutput>
<cfabort>
</cfif>
if (arrayLen(uploadedFiles) == 0) {
throw(message="No files uploaded");
}
<cftry>
<!--- Check for uploaded files --->
<cfset uploadedFiles = arrayNew(1)>
// Process uploaded files - convert to base64
imageDataArray = [];
<!--- Get form fields for file uploads (file0, file1, file2, etc.) --->
<cfset formFields = form.keyArray()>
<cfloop array="#formFields#" index="fieldName">
<cfif reFindNoCase("^file[0-9]+$", fieldName) AND len(form[fieldName])>
<cfset arrayAppend(uploadedFiles, fieldName)>
</cfif>
</cfloop>
// Create temp directory for uploads if needed
uploadDir = getTempDirectory() & "payfrit_menu_uploads/";
if (!directoryExists(uploadDir)) {
directoryCreate(uploadDir);
}
<cfif arrayLen(uploadedFiles) EQ 0>
<cfthrow message="No files uploaded">
</cfif>
for (fieldName in uploadedFiles) {
try {
// Upload the file
uploadResult = fileUpload(
uploadDir,
fieldName,
"image/jpeg,image/png,image/gif,image/webp,application/pdf",
"makeunique"
);
} catch (any uploadErr) {
throw(message="File upload failed: " & uploadErr.message, detail="Field: " & fieldName);
}
<!--- Process uploaded files - convert to base64 --->
<cfset imageDataArray = arrayNew(1)>
if (uploadResult.fileWasSaved) {
filePath = uploadResult.serverDirectory & "/" & uploadResult.serverFile;
fileExt = lCase(uploadResult.serverFileExt);
<!--- Create temp directory for uploads if needed --->
<cfset uploadDir = getTempDirectory() & "payfrit_menu_uploads/">
<cfif NOT directoryExists(uploadDir)>
<cfdirectory action="create" directory="#uploadDir#">
</cfif>
// For images, resize if too large (max 1600px on longest side)
if (listFindNoCase("jpg,jpeg,png,gif,webp", fileExt)) {
try {
img = imageRead(filePath);
} catch (any readErr) {
throw(message="Image read failed: " & readErr.message, detail="File: " & filePath);
}
<cfloop array="#uploadedFiles#" index="fieldName">
<!--- Upload the file --->
<cffile action="upload" destination="#uploadDir#" filefield="#fieldName#"
accept="image/jpeg,image/png,image/gif,image/webp,application/pdf"
nameconflict="makeunique" result="uploadResult">
imgWidth = img.width;
imgHeight = img.height;
maxDimension = 1600;
<cfif uploadResult.fileWasSaved>
<cfset filePath = uploadResult.serverDirectory & "/" & uploadResult.serverFile>
<cfset fileExt = lCase(uploadResult.serverFileExt)>
if (imgWidth > maxDimension || imgHeight > maxDimension) {
try {
if (imgWidth > imgHeight) {
newWidth = maxDimension;
newHeight = int(imgHeight * (maxDimension / imgWidth));
} else {
newHeight = maxDimension;
newWidth = int(imgWidth * (maxDimension / imgHeight));
}
imageResize(img, newWidth, newHeight);
} catch (any resizeErr) {
throw(message="Image resize failed: " & resizeErr.message, detail="Dimensions: " & imgWidth & "x" & imgHeight);
}
}
// Re-save with good quality compression
try {
imageWrite(img, filePath, 0.8);
} catch (any writeErr) {
throw(message="Image write failed: " & writeErr.message, detail="File: " & filePath);
}
}
<!--- For images, resize if too large (max 1600px on longest side) --->
<cfif listFindNoCase("jpg,jpeg,png,gif,webp", fileExt)>
<cfimage action="read" source="#filePath#" name="img">
<cfset imgWidth = img.width>
<cfset imgHeight = img.height>
<cfset maxDimension = 1600>
// Read file and convert to base64
fileContent = fileReadBinary(filePath);
base64Content = toBase64(fileContent);
<cfif imgWidth GT maxDimension OR imgHeight GT maxDimension>
<cfif imgWidth GT imgHeight>
<cfset newWidth = maxDimension>
<cfset newHeight = int(imgHeight * (maxDimension / imgWidth))>
<cfelse>
<cfset newHeight = maxDimension>
<cfset newWidth = int(imgWidth * (maxDimension / imgHeight))>
</cfif>
<cfimage action="resize" source="#img#" width="#newWidth#" height="#newHeight#" name="img">
</cfif>
<!--- Re-save with good quality compression --->
<cfimage action="write" source="#img#" destination="#filePath#" quality="0.8" overwrite="true">
</cfif>
// Determine media type
mediaType = "image/jpeg";
if (fileExt == "png") mediaType = "image/png";
else if (fileExt == "gif") mediaType = "image/gif";
else if (fileExt == "webp") mediaType = "image/webp";
else if (fileExt == "pdf") mediaType = "application/pdf";
<!--- Read file and convert to base64 --->
<cffile action="readbinary" file="#filePath#" variable="fileContent">
<cfset base64Content = toBase64(fileContent)>
// Claude API uses different structure for PDFs vs images
if (fileExt == "pdf") {
arrayAppend(imageDataArray, {
"type": "document",
"source": {
"type": "base64",
"media_type": "application/pdf",
"data": base64Content
}
});
} else {
arrayAppend(imageDataArray, {
"type": "image",
"source": {
"type": "base64",
"media_type": mediaType,
"data": base64Content
}
});
}
<!--- Determine media type --->
<cfset mediaType = "image/jpeg">
<cfif fileExt EQ "png">
<cfset mediaType = "image/png">
<cfelseif fileExt EQ "gif">
<cfset mediaType = "image/gif">
<cfelseif fileExt EQ "webp">
<cfset mediaType = "image/webp">
<cfelseif fileExt EQ "pdf">
<cfset mediaType = "application/pdf">
</cfif>
// Clean up temp file
fileDelete(filePath);
}
}
<!--- Claude API uses different structure for PDFs vs images --->
<cfif fileExt EQ "pdf">
<cfset imgStruct = structNew()>
<cfset imgStruct["type"] = "document">
<cfset imgStruct["source"] = structNew()>
<cfset imgStruct["source"]["type"] = "base64">
<cfset imgStruct["source"]["media_type"] = "application/pdf">
<cfset imgStruct["source"]["data"] = base64Content>
<cfset arrayAppend(imageDataArray, imgStruct)>
<cfelse>
<cfset imgStruct = structNew()>
<cfset imgStruct["type"] = "image">
<cfset imgStruct["source"] = structNew()>
<cfset imgStruct["source"]["type"] = "base64">
<cfset imgStruct["source"]["media_type"] = mediaType>
<cfset imgStruct["source"]["data"] = base64Content>
<cfset arrayAppend(imageDataArray, imgStruct)>
</cfif>
if (arrayLen(imageDataArray) == 0) {
throw(message="No valid images could be processed");
}
<!--- Clean up temp file --->
<cffile action="delete" file="#filePath#">
</cfif>
</cfloop>
// Build the prompt for Claude
systemPrompt = "You are an expert at extracting structured menu data from restaurant menu images. Your task is to analyze menu images and return the data in a specific JSON format. Be thorough and extract ALL items, categories, and any modifier patterns you can identify.";
<cfif arrayLen(imageDataArray) EQ 0>
<cfthrow message="No valid images could be processed">
</cfif>
userPrompt = "Please analyze these menu images and extract all the information into a JSON object with these keys: business (with name, address, phone, hours), categories (array with name and itemCount), modifiers (array with name, required boolean, and options array), and items (array with name, description, price, category, and modifiers array). Extract EVERY menu item with accurate prices. Group items by category. Look for modifier patterns like sizes, protein choices, bread choices, and add-ons. Create reusable modifier templates for patterns that appear on multiple items. Return ONLY valid JSON, no other text.";
<!--- Build the prompt for Claude --->
<cfset systemPrompt = "You are an expert at extracting structured menu data from restaurant menu images. Your task is to analyze menu images and return the data in a specific JSON format. Be thorough and extract ALL items, categories, and any modifier patterns you can identify.">
// Build the messages array with images
messagesContent = [];
<cfset userPrompt = "Please analyze these menu images and extract all the information into a JSON object with these keys: business (with name, address, phone, hours), categories (array with name and itemCount), modifiers (array with name, required boolean, and options array), and items (array with name, description, price, category, and modifiers array). Extract EVERY menu item with accurate prices. Group items by category. Look for modifier patterns like sizes, protein choices, bread choices, and add-ons. Create reusable modifier templates for patterns that appear on multiple items. Return ONLY valid JSON, no other text.">
// Add each image
for (imgData in imageDataArray) {
arrayAppend(messagesContent, imgData);
}
<!--- Build the messages array with images --->
<cfset messagesContent = arrayNew(1)>
// Add the text prompt
arrayAppend(messagesContent, {
"type": "text",
"text": userPrompt
});
<!--- Add each image --->
<cfloop array="#imageDataArray#" index="imgData">
<cfset arrayAppend(messagesContent, imgData)>
</cfloop>
// Build request body
requestBody = {
"model": "claude-sonnet-4-20250514",
"max_tokens": 8192,
"system": systemPrompt,
"messages": [
{
"role": "user",
"content": messagesContent
}
]
};
<!--- Add the text prompt --->
<cfset textBlock = structNew()>
<cfset textBlock["type"] = "text">
<cfset textBlock["text"] = userPrompt>
<cfset arrayAppend(messagesContent, textBlock)>
// Write request body to temp file (Lucee's HTTP client has issues with large payloads)
tempRequestFile = getTempDirectory() & "claude_request_" & createUUID() & ".json";
tempResponseFile = getTempDirectory() & "claude_response_" & createUUID() & ".json";
fileWrite(tempRequestFile, serializeJSON(requestBody));
<!--- Build request body --->
<cfset userMessage = structNew()>
<cfset userMessage["role"] = "user">
<cfset userMessage["content"] = messagesContent>
try {
// Call Claude API using cfhttp tag (more reliable than http component)
httpResult = {};
http url="https://api.anthropic.com/v1/messages" method="POST" timeout=300 result="httpResult" {
httpparam type="header" name="Content-Type" value="application/json";
httpparam type="header" name="x-api-key" value=CLAUDE_API_KEY;
httpparam type="header" name="anthropic-version" value="2023-06-01";
httpparam type="body" value=serializeJSON(requestBody);
}
<cfset requestBody = structNew()>
<cfset requestBody["model"] = "claude-sonnet-4-20250514">
<cfset requestBody["max_tokens"] = 8192>
<cfset requestBody["system"] = systemPrompt>
<cfset requestBody["messages"] = arrayNew(1)>
<cfset arrayAppend(requestBody["messages"], userMessage)>
httpStatusCode = httpResult.statusCode ?: "0";
httpFileContent = httpResult.fileContent ?: "";
<!--- Call Claude API using Java HttpURLConnection (bypasses Lucee's HTTP client issues) --->
<cfset requestJson = serializeJSON(requestBody)>
<cfset httpFileContent = "">
<cfset httpStatusCode = 0>
// Normalize status code
if (isNumeric(httpStatusCode)) {
httpStatusCode = int(httpStatusCode);
} else if (findNoCase("200", httpStatusCode)) {
httpStatusCode = 200;
}
<cftry>
<cfset urlObj = createObject("java", "java.net.URL").init("https://api.anthropic.com/v1/messages")>
<cfset conn = urlObj.openConnection()>
<cfset conn.setRequestMethod("POST")>
<cfset conn.setDoOutput(true)>
<cfset conn.setConnectTimeout(30000)>
<cfset conn.setReadTimeout(300000)>
<cfset conn.setRequestProperty("Content-Type", "application/json")>
<cfset conn.setRequestProperty("x-api-key", CLAUDE_API_KEY)>
<cfset conn.setRequestProperty("anthropic-version", "2023-06-01")>
if (httpStatusCode != 200) {
// Try to parse error message from Claude's response
errorMsg = "Claude API error: " & httpResult.statusCode;
try {
errorData = deserializeJSON(httpFileContent);
if (structKeyExists(errorData, "error") && structKeyExists(errorData.error, "message")) {
errorMsg = errorMsg & " - " & errorData.error.message;
}
} catch (any e) {
// Use raw response if can't parse
}
throw(message=errorMsg, detail=httpFileContent);
}
} finally {
// Clean up temp files
if (fileExists(tempRequestFile)) fileDelete(tempRequestFile);
}
<!--- Write request body --->
<cfset outputStream = conn.getOutputStream()>
<cfset writer = createObject("java", "java.io.OutputStreamWriter").init(outputStream, "UTF-8")>
<cfset writer.write(requestJson)>
<cfset writer.flush()>
<cfset writer.close()>
// Parse Claude's response
claudeResponse = deserializeJSON(httpFileContent);
<!--- Get response --->
<cfset httpStatusCode = conn.getResponseCode()>
if (!structKeyExists(claudeResponse, "content") || !arrayLen(claudeResponse.content)) {
throw(message="Empty response from Claude");
}
<!--- Read response body --->
<cfif httpStatusCode GTE 200 AND httpStatusCode LT 300>
<cfset inputStream = conn.getInputStream()>
<cfelse>
<cfset inputStream = conn.getErrorStream()>
</cfif>
// Extract the text content
responseText = "";
for (block in claudeResponse.content) {
if (block.type == "text") {
responseText = block.text;
break;
}
}
<cfif NOT isNull(inputStream)>
<cfset reader = createObject("java", "java.io.BufferedReader").init(
createObject("java", "java.io.InputStreamReader").init(inputStream, "UTF-8")
)>
<cfset sb = createObject("java", "java.lang.StringBuilder").init()>
<cfset line = reader.readLine()>
<cfloop condition="NOT isNull(line)">
<cfset sb.append(line)>
<cfset line = reader.readLine()>
</cfloop>
<cfset reader.close()>
<cfset httpFileContent = sb.toString()>
</cfif>
// Parse the JSON from Claude's response
// Claude might wrap it in markdown code blocks, so strip those
responseText = trim(responseText);
if (left(responseText, 7) == "```json") {
responseText = mid(responseText, 8, len(responseText) - 7);
}
if (left(responseText, 3) == "```") {
responseText = mid(responseText, 4, len(responseText) - 3);
}
if (right(responseText, 3) == "```") {
responseText = left(responseText, len(responseText) - 3);
}
responseText = trim(responseText);
<cfset conn.disconnect()>
extractedData = deserializeJSON(responseText);
<cfcatch type="any">
<cfset httpStatusCode = 0>
<cfset errorStruct = structNew()>
<cfset errorStruct["error"] = structNew()>
<cfset errorStruct["error"]["message"] = cfcatch.message>
<cfset httpFileContent = serializeJSON(errorStruct)>
</cfcatch>
</cftry>
// Update category item counts
if (structKeyExists(extractedData, "categories") && structKeyExists(extractedData, "items")) {
for (i = 1; i <= arrayLen(extractedData.categories); i++) {
catName = extractedData.categories[i].name;
itemCount = 0;
for (item in extractedData.items) {
if (structKeyExists(item, "category") && item.category == catName) {
itemCount++;
}
}
extractedData.categories[i].itemCount = itemCount;
}
}
<!--- Normalize status code --->
<cfif isNumeric(httpStatusCode)>
<cfset httpStatusCode = int(httpStatusCode)>
<cfelseif findNoCase("200", httpStatusCode)>
<cfset httpStatusCode = 200>
</cfif>
// Add unique IDs to items
if (structKeyExists(extractedData, "items")) {
for (i = 1; i <= arrayLen(extractedData.items); i++) {
extractedData.items[i].id = "item_" & i;
}
}
<cfif httpStatusCode NEQ 200>
<!--- Try to parse error message from Claude's response --->
<cfset errorMsg = "Claude API error: " & httpStatusCode>
<cftry>
<cfset errorData = deserializeJSON(httpFileContent)>
<cfif structKeyExists(errorData, "error") AND structKeyExists(errorData.error, "message")>
<cfset errorMsg = errorMsg & " - " & errorData.error.message>
</cfif>
<cfcatch type="any">
<!--- Use raw response if can't parse --->
</cfcatch>
</cftry>
<cfthrow message="#errorMsg#" detail="#httpFileContent#">
</cfif>
response.OK = true;
response.DATA = extractedData;
response.imagesProcessed = arrayLen(imageDataArray);
<!--- Parse Claude's response --->
<cfset claudeResponse = deserializeJSON(httpFileContent)>
} catch (any e) {
response.MESSAGE = e.message;
if (len(e.detail)) {
response.DETAIL = e.detail;
}
// Add debug info
if (structKeyExists(e, "tagContext") && arrayLen(e.tagContext) > 0) {
response.DEBUG_LINE = e.tagContext[1].line;
response.DEBUG_TEMPLATE = e.tagContext[1].template;
}
}
<cfif NOT structKeyExists(claudeResponse, "content") OR NOT arrayLen(claudeResponse.content)>
<cfthrow message="Empty response from Claude">
</cfif>
writeOutput(serializeJSON(response));
</cfscript>
<!--- Extract the text content --->
<cfset responseText = "">
<cfloop array="#claudeResponse.content#" index="block">
<cfif block.type EQ "text">
<cfset responseText = block.text>
<cfbreak>
</cfif>
</cfloop>
<!--- Parse the JSON from Claude's response --->
<!--- Claude might wrap it in markdown code blocks, so strip those --->
<cfset responseText = trim(responseText)>
<cfif left(responseText, 7) EQ "```json">
<cfset responseText = mid(responseText, 8, len(responseText) - 7)>
</cfif>
<cfif left(responseText, 3) EQ "```">
<cfset responseText = mid(responseText, 4, len(responseText) - 3)>
</cfif>
<cfif right(responseText, 3) EQ "```">
<cfset responseText = left(responseText, len(responseText) - 3)>
</cfif>
<cfset responseText = trim(responseText)>
<cfset extractedData = deserializeJSON(responseText)>
<!--- Update category item counts --->
<cfif structKeyExists(extractedData, "categories") AND structKeyExists(extractedData, "items")>
<cfloop from="1" to="#arrayLen(extractedData.categories)#" index="i">
<cfset catName = extractedData.categories[i].name>
<cfset itemCount = 0>
<cfloop array="#extractedData.items#" index="item">
<cfif structKeyExists(item, "category") AND item.category EQ catName>
<cfset itemCount = itemCount + 1>
</cfif>
</cfloop>
<cfset extractedData.categories[i]["itemCount"] = itemCount>
</cfloop>
</cfif>
<!--- Add unique IDs to items --->
<cfif structKeyExists(extractedData, "items")>
<cfloop from="1" to="#arrayLen(extractedData.items)#" index="i">
<cfset extractedData.items[i]["id"] = "item_" & i>
</cfloop>
</cfif>
<cfset response["OK"] = true>
<cfset response["DATA"] = extractedData>
<cfset response["imagesProcessed"] = arrayLen(imageDataArray)>
<cfcatch type="any">
<cfset response["MESSAGE"] = cfcatch.message>
<cfif len(cfcatch.detail)>
<cfset response["DETAIL"] = cfcatch.detail>
</cfif>
<cfif structKeyExists(cfcatch, "tagContext") AND arrayLen(cfcatch.tagContext) GT 0>
<cfset response["DEBUG_LINE"] = cfcatch.tagContext[1].line>
<cfset response["DEBUG_TEMPLATE"] = cfcatch.tagContext[1].template>
</cfif>
</cfcatch>
</cftry>
<cfoutput>#serializeJSON(response)#</cfoutput>