payfrit-works/api/setup/analyzeMenuImages.cfm

<cfsetting showdebugoutput="false">
<cfsetting enablecfoutputonly="true">
<cfcontent type="application/json; charset=utf-8" reset="true">

<cfscript>
/**
 * Analyze Menu Images using Claude Vision API
 *
 * Accepts uploaded menu images (JPG, PNG, PDF) and sends them to Claude
 * to extract structured menu data including:
 * - Business info (name, address, phone, hours)
 * - Categories
 * - Modifier templates
 * - Menu items with prices and descriptions
 *
 * Returns structured JSON for the setup wizard to display
 */

response = { "OK": false };

// Claude API key - should be in environment or config
CLAUDE_API_KEY = application.claudeApiKey ?: "";

// If not in application scope, try to read from config
if (!len(CLAUDE_API_KEY)) {
    try {
        configPath = expandPath("/biz.payfrit.com/config/claude.json");
        if (fileExists(configPath)) {
            configData = deserializeJSON(fileRead(configPath));
            CLAUDE_API_KEY = configData.apiKey ?: "";
        }
    } catch (any e) {
        // Config file doesn't exist or is invalid
    }
}

if (!len(CLAUDE_API_KEY)) {
    response.MESSAGE = "Claude API key not configured";
    writeOutput(serializeJSON(response));
    abort;
}

try {
    // Check for uploaded files
    uploadedFiles = [];

    // Get form fields for file uploads (file0, file1, file2, etc.)
    formFields = form.keyArray();
    for (fieldName in formFields) {
        if (reFindNoCase("^file\d+$", fieldName) && len(form[fieldName])) {
            // This is a file field
            arrayAppend(uploadedFiles, fieldName);
        }
    }

    if (arrayLen(uploadedFiles) == 0) {
        throw(message="No files uploaded");
    }

    // Process uploaded files - convert to base64
    imageDataArray = [];

    // Create temp directory for uploads if needed
    uploadDir = getTempDirectory() & "payfrit_menu_uploads/";
    if (!directoryExists(uploadDir)) {
        directoryCreate(uploadDir);
    }

    for (fieldName in uploadedFiles) {
        // Upload the file
        uploadResult = fileUpload(
            uploadDir,
            fieldName,
            "image/jpeg,image/png,image/gif,image/webp,application/pdf",
            "makeunique"
        );

        if (uploadResult.fileWasSaved) {
            filePath = uploadResult.serverDirectory & "/" & uploadResult.serverFile;
            fileExt = lCase(uploadResult.serverFileExt);

            // Read file and convert to base64
            fileContent = fileReadBinary(filePath);
            base64Content = toBase64(fileContent);

            // Determine media type
            mediaType = "image/jpeg";
            if (fileExt == "png") mediaType = "image/png";
            else if (fileExt == "gif") mediaType = "image/gif";
            else if (fileExt == "webp") mediaType = "image/webp";
            else if (fileExt == "pdf") mediaType = "application/pdf";

            arrayAppend(imageDataArray, {
                "type": "image",
                "source": {
                    "type": "base64",
                    "media_type": mediaType,
                    "data": base64Content
                }
            });

            // Clean up temp file
            fileDelete(filePath);
        }
    }

    if (arrayLen(imageDataArray) == 0) {
        throw(message="No valid images could be processed");
    }

    // Build the prompt for Claude
    systemPrompt = "You are an expert at extracting structured menu data from restaurant menu images. Your task is to analyze menu images and return the data in a specific JSON format. Be thorough and extract ALL items, categories, and any modifier patterns you can identify.";

    userPrompt = "Please analyze these menu images and extract all the information into the following JSON structure:

{
  ""business"": {
    ""name"": ""Restaurant name if visible"",
    ""address"": ""Full address if visible"",
    ""phone"": ""Phone number(s) if visible"",
    ""hours"": ""Business hours if visible""
  },
  ""categories"": [
    {
      ""name"": ""Category name"",
      ""itemCount"": 0
    }
  ],
  ""modifiers"": [
    {
      ""name"": ""Modifier template name (e.g., 'Size', 'Protein Choice', 'Toast Type')"",
      ""required"": true or false,
      ""options"": [
        { ""name"": ""Option name"", ""price"": 0.00 }
      ]
    }
  ],
  ""items"": [
    {
      ""name"": ""Item name"",
      ""description"": ""Item description if any"",
      ""price"": 0.00,
      ""category"": ""Category this item belongs to"",
      ""modifiers"": [""Name of applicable modifier template""]
    }
  ]
}

Important guidelines:
1. Extract EVERY menu item you can see, with accurate prices
2. Group items into their categories as shown on the menu
3. Look for modifier patterns like:
   - Size options (Small/Regular/Large)
   - Protein choices (Chicken/Beef/Shrimp)
   - Bread choices (Wheat/Sourdough/White/Rye)
   - Add-ons with prices (Add bacon +$2.00)
   - Required choices (served with choice of side)
4. For modifiers that appear on multiple items, create a reusable template
5. Assign the appropriate modifier templates to each item
6. If a price has a '+' it's usually an upcharge modifier option
7. Pay attention to category headers and section dividers

Return ONLY the JSON object, no other text.";

    // Build the messages array with images
    messagesContent = [];

    // Add each image
    for (imgData in imageDataArray) {
        arrayAppend(messagesContent, imgData);
    }

    // Add the text prompt
    arrayAppend(messagesContent, {
        "type": "text",
        "text": userPrompt
    });

    // Call Claude API
    httpService = new http();
    httpService.setMethod("POST");
    httpService.setUrl("https://api.anthropic.com/v1/messages");
    httpService.addParam(type="header", name="Content-Type", value="application/json");
    httpService.addParam(type="header", name="x-api-key", value=CLAUDE_API_KEY);
    httpService.addParam(type="header", name="anthropic-version", value="2023-06-01");

    requestBody = {
        "model": "claude-sonnet-4-20250514",
        "max_tokens": 8192,
        "system": systemPrompt,
        "messages": [
            {
                "role": "user",
                "content": messagesContent
            }
        ]
    };

    httpService.addParam(type="body", value=serializeJSON(requestBody));

    httpResult = httpService.send().getPrefix();

    if (httpResult.statusCode != "200 OK" && httpResult.statusCode != 200) {
        throw(message="Claude API error: " & httpResult.statusCode, detail=httpResult.fileContent);
    }

    // Parse Claude's response
    claudeResponse = deserializeJSON(httpResult.fileContent);

    if (!structKeyExists(claudeResponse, "content") || !arrayLen(claudeResponse.content)) {
        throw(message="Empty response from Claude");
    }

    // Extract the text content
    responseText = "";
    for (block in claudeResponse.content) {
        if (block.type == "text") {
            responseText = block.text;
            break;
        }
    }

    // Parse the JSON from Claude's response
    // Claude might wrap it in markdown code blocks, so strip those
    responseText = trim(responseText);
    if (left(responseText, 7) == "```json") {
        responseText = mid(responseText, 8, len(responseText) - 7);
    }
    if (left(responseText, 3) == "```") {
        responseText = mid(responseText, 4, len(responseText) - 3);
    }
    if (right(responseText, 3) == "```") {
        responseText = left(responseText, len(responseText) - 3);
    }
    responseText = trim(responseText);

    extractedData = deserializeJSON(responseText);

    // Update category item counts
    if (structKeyExists(extractedData, "categories") && structKeyExists(extractedData, "items")) {
        for (i = 1; i <= arrayLen(extractedData.categories); i++) {
            catName = extractedData.categories[i].name;
            itemCount = 0;
            for (item in extractedData.items) {
                if (structKeyExists(item, "category") && item.category == catName) {
                    itemCount++;
                }
            }
            extractedData.categories[i].itemCount = itemCount;
        }
    }

    // Add unique IDs to items
    if (structKeyExists(extractedData, "items")) {
        for (i = 1; i <= arrayLen(extractedData.items); i++) {
            extractedData.items[i].id = "item_" & i;
        }
    }

    response.OK = true;
    response.DATA = extractedData;
    response.imagesProcessed = arrayLen(imageDataArray);

} catch (any e) {
    response.MESSAGE = e.message;
    if (len(e.detail)) {
        response.DETAIL = e.detail;
    }
}

writeOutput(serializeJSON(response));
</cfscript>