From 9e195b79e0e53a9f1ef2365fd56a75de1b5b1f75 Mon Sep 17 00:00:00 2001
From: John Mizerek <mizerek@gmail.com>
Date: Thu, 15 Jan 2026 15:29:15 -0800
Subject: [PATCH] Add conversational modifier assignment for uncertain
 modifiers

Backend (analyzeMenuImages.cfm):
- Updated AI prompts to classify modifiers by confidence level
- Added appliesTo field: 'category', 'item', or 'uncertain'
- Added categoryName field for category-level modifiers
- Auto-assign category-level modifiers to items in that category
- Only assign item-level modifiers when clearly in item description
- Mark uncertain modifiers for user confirmation

Frontend (setup-wizard.html):
- Added showUncertainModifiersStep() between modifier and item steps
- Shows conversational prompts for each uncertain modifier
- Users can select which categories each modifier applies to
- Users can skip modifiers that don't apply automatically
- Applies user selections to items before proceeding
- Added CSS styling for category selection checkboxes

Flow:
1. AI extracts modifiers with confidence classification
2. Category-level modifiers auto-assigned to items
3. Uncertain modifiers presented one-by-one for user decision
4. User confirms or skips each uncertain modifier
5. Assignments applied to items before save

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
---
 api/setup/analyzeMenuImages.cfm |  42 ++++++++++-
 portal/setup-wizard.html        | 130 +++++++++++++++++++++++++++++++-
 2 files changed, 169 insertions(+), 3 deletions(-)

diff --git a/api/setup/analyzeMenuImages.cfm b/api/setup/analyzeMenuImages.cfm
index 7a0525c..66782d0 100644
--- a/api/setup/analyzeMenuImages.cfm
+++ b/api/setup/analyzeMenuImages.cfm
@@ -94,7 +94,7 @@
     </cfif>
 
     <!--- System prompt for per-image analysis --->
-    <cfset systemPrompt = "You are an expert at extracting structured menu data from restaurant menu images. Extract ALL data visible on THIS image only. Return valid JSON with these keys: business (object with name, address, phone, hours - only if visible), categories (array of category names visible), modifiers (array of modifier templates with name, required boolean, and options array where each option is an object with 'name' and 'price' keys), items (array with name, description, price, category, and modifiers array). For modifier options, ALWAYS use format: {""name"": ""option name"", ""price"": 0}. IMPORTANT: Look for modifiers in these locations: (1) text under category headers (e.g., 'All burgers include...', 'Choose your size:'), (2) item descriptions (e.g., 'served with choice of side'), (3) asterisk notes, (4) header/footer text. For each item, include a 'modifiers' array with names of modifier templates that apply based on: category-level modifiers, item description mentions, or obvious context. Only include modifiers when confident - when unsure, omit them. Return ONLY valid JSON, no markdown, no explanation.">
+    <cfset systemPrompt = "You are an expert at extracting structured menu data from restaurant menu images. Extract ALL data visible on THIS image only. Return valid JSON with these keys: business (object with name, address, phone, hours - only if visible), categories (array of category names visible), modifiers (array of modifier templates with name, required boolean, appliesTo (string: 'category', 'item', or 'uncertain'), categoryName (if appliesTo='category'), and options array where each option is an object with 'name' and 'price' keys), items (array with name, description, price, category, and modifiers array). For modifier options, ALWAYS use format: {""name"": ""option name"", ""price"": 0}. IMPORTANT: Look for modifiers in these locations: (1) text under category headers (e.g., 'All burgers include...', 'Choose your size:'), (2) item descriptions (e.g., 'served with choice of side'), (3) asterisk notes, (4) header/footer text. For modifiers found under category headers, set appliesTo='category' and categoryName to the category. For modifiers clearly in item descriptions, assign them to that item's modifiers array. For modifiers where you're uncertain how they apply, set appliesTo='uncertain' and do NOT assign to items. Return ONLY valid JSON, no markdown, no explanation.">
 
     <!--- Process each image individually --->
     <cfset allResults = arrayNew(1)>
@@ -108,7 +108,7 @@
 
         <cfset textBlock = structNew()>
         <cfset textBlock["type"] = "text">
-        <cfset textBlock["text"] = "Extract all menu data from this image. Return JSON with: business (if visible), categories, modifiers (with options as objects with name and price keys), items (with modifiers array). Look for modifiers under category headers, in item descriptions, in notes/asterisks, or in headers/footers. Assign modifiers to items based on: category-level rules (e.g., 'all burgers include...'), item descriptions, or obvious context. Example item: {""name"": ""Cheeseburger"", ""price"": 12.99, ""category"": ""Burgers"", ""modifiers"": [""Size"", ""Toppings""]}. Example modifier: {""name"": ""Size"", ""required"": true, ""options"": [{""name"": ""Small"", ""price"": 0}, {""name"": ""Large"", ""price"": 1.50}]}">
+        <cfset textBlock["text"] = "Extract all menu data from this image. Return JSON with: business (if visible), categories, modifiers (with appliesTo, categoryName if applicable, and options as objects with name and price keys), items (with modifiers array only for item-specific modifiers). Look for modifiers under category headers, in item descriptions, in notes/asterisks, or in headers/footers. Example category-level modifier: {""name"": ""Size"", ""required"": true, ""appliesTo"": ""category"", ""categoryName"": ""Burgers"", ""options"": [{""name"": ""Small"", ""price"": 0}, {""name"": ""Large"", ""price"": 1.50}]}. Example uncertain modifier: {""name"": ""Toppings"", ""required"": false, ""appliesTo"": ""uncertain"", ""options"": [{""name"": ""Lettuce"", ""price"": 0}, {""name"": ""Tomato"", ""price"": 0}]}. Only assign modifiers to item.modifiers array when the modifier is clearly specific to that individual item from its description.">
         <cfset arrayAppend(messagesContent, textBlock)>
 
         <cfset userMessage = structNew()>
@@ -233,6 +233,10 @@
                         <cfset normalizedMod = structNew()>
                         <cfset normalizedMod["name"] = trim(mod.name)>
                         <cfset normalizedMod["required"] = structKeyExists(mod, "required") AND mod.required EQ true>
+                        <cfset normalizedMod["appliesTo"] = structKeyExists(mod, "appliesTo") ? mod.appliesTo : "uncertain">
+                        <cfif structKeyExists(mod, "categoryName") AND len(trim(mod.categoryName))>
+                            <cfset normalizedMod["categoryName"] = trim(mod.categoryName)>
+                        </cfif>
                         <cfset normalizedMod["options"] = arrayNew(1)>
 
                         <!--- Normalize options array --->
@@ -302,6 +306,40 @@
         </cfif>
     </cfloop>
 
+    <!--- 5. Auto-assign category-level modifiers to items --->
+    <cfloop array="#mergedItems#" index="item">
+        <!--- Ensure item has modifiers array --->
+        <cfif NOT structKeyExists(item, "modifiers") OR NOT isArray(item.modifiers)>
+            <cfset item["modifiers"] = arrayNew(1)>
+        </cfif>
+
+        <!--- Get item's category --->
+        <cfset itemCategory = structKeyExists(item, "category") ? trim(item.category) : "">
+
+        <!--- Find category-level modifiers for this category --->
+        <cfif len(itemCategory)>
+            <cfloop array="#mergedModifiers#" index="mod">
+                <cfif structKeyExists(mod, "appliesTo") AND mod.appliesTo EQ "category" AND structKeyExists(mod, "categoryName")>
+                    <!--- Case-insensitive category match --->
+                    <cfif lCase(mod.categoryName) EQ lCase(itemCategory)>
+                        <!--- Check if modifier is not already assigned --->
+                        <cfset alreadyAssigned = false>
+                        <cfloop array="#item.modifiers#" index="existingMod">
+                            <cfif lCase(existingMod) EQ lCase(mod.name)>
+                                <cfset alreadyAssigned = true>
+                                <cfbreak>
+                            </cfif>
+                        </cfloop>
+
+                        <cfif NOT alreadyAssigned>
+                            <cfset arrayAppend(item.modifiers, mod.name)>
+                        </cfif>
+                    </cfif>
+                </cfif>
+            </cfloop>
+        </cfif>
+    </cfloop>
+
     <!--- Build final response --->
     <cfset finalData = structNew()>
     <cfset finalData["business"] = mergedBusiness>
diff --git a/portal/setup-wizard.html b/portal/setup-wizard.html
index 3d8286e..a56bc13 100644
--- a/portal/setup-wizard.html
+++ b/portal/setup-wizard.html
@@ -335,6 +335,33 @@
       margin-left: 4px;
     }
 
+    /* Category Selection */
+    .category-option {
+      display: flex;
+      align-items: center;
+      gap: 8px;
+      padding: 12px;
+      background: #fff;
+      border: 1px solid var(--gray-200);
+      border-radius: 6px;
+      cursor: pointer;
+      transition: all 0.2s;
+    }
+
+    .category-option:hover {
+      border-color: var(--primary);
+      background: rgba(99, 102, 241, 0.05);
+    }
+
+    .category-option input[type="checkbox"] {
+      cursor: pointer;
+    }
+
+    .category-option span {
+      font-size: 14px;
+      color: var(--gray-700);
+    }
+
     /* Action Buttons */
     .action-buttons {
       display: flex;
@@ -1453,7 +1480,108 @@
       });
 
       config.extractedData.modifiers = updatedModifiers;
-      showItemsStep();
+
+      // Check if there are uncertain modifiers that need category assignment
+      showUncertainModifiersStep();
+    }
+
+    // New step: Handle uncertain modifier assignments
+    function showUncertainModifiersStep() {
+      const modifiers = config.extractedData.modifiers || [];
+      const categories = config.extractedData.categories || [];
+
+      // Find modifiers marked as "uncertain"
+      const uncertainModifiers = modifiers.filter(mod =>
+        mod.appliesTo === 'uncertain'
+      );
+
+      if (uncertainModifiers.length === 0 || categories.length === 0) {
+        // No uncertain modifiers or no categories, skip to items
+        showItemsStep();
+        return;
+      }
+
+      // Initialize uncertain modifier assignment tracking
+      if (!config.uncertainModifierAssignments) {
+        config.uncertainModifierAssignments = {};
+        config.currentUncertainModIndex = 0;
+      }
+
+      const currentIndex = config.currentUncertainModIndex;
+
+      if (currentIndex >= uncertainModifiers.length) {
+        // All uncertain modifiers have been processed, apply assignments and continue
+        applyUncertainModifierAssignments();
+        showItemsStep();
+        return;
+      }
+
+      const modifier = uncertainModifiers[currentIndex];
+
+      // Ask user about this modifier
+      const categoryOptions = categories.map((cat, i) => `
+        <label class="category-option">
+          <input type="checkbox" name="category-assign" value="${cat.name}">
+          <span>${cat.name}</span>
+        </label>
+      `).join('');
+
+      addMessage('ai', `
+        <p>I found the modifier template <strong>"${modifier.name}"</strong> but I'm not sure which items it applies to.</p>
+        <p>Select the categories where this modifier should be applied, or skip if it doesn't apply automatically:</p>
+        <div class="category-selection" style="display: flex; flex-direction: column; gap: 8px; margin: 16px 0;">
+          ${categoryOptions}
+        </div>
+        <div class="action-buttons">
+          <button class="btn btn-outline" onclick="skipUncertainModifier()">Skip This Modifier</button>
+          <button class="btn btn-primary" onclick="assignUncertainModifier('${modifier.name}')">Apply to Selected Categories</button>
+        </div>
+      `);
+    }
+
+    function skipUncertainModifier() {
+      config.currentUncertainModIndex++;
+      showUncertainModifiersStep();
+    }
+
+    function assignUncertainModifier(modifierName) {
+      const checkboxes = document.querySelectorAll('input[name="category-assign"]:checked');
+      const selectedCategories = Array.from(checkboxes).map(cb => cb.value);
+
+      if (selectedCategories.length > 0) {
+        config.uncertainModifierAssignments[modifierName] = selectedCategories;
+      }
+
+      config.currentUncertainModIndex++;
+      showUncertainModifiersStep();
+    }
+
+    function applyUncertainModifierAssignments() {
+      // Apply the user's category selections to items
+      const items = config.extractedData.items || [];
+      const modifiers = config.extractedData.modifiers || [];
+
+      for (const [modifierName, categories] of Object.entries(config.uncertainModifierAssignments)) {
+        items.forEach(item => {
+          if (!item.modifiers) {
+            item.modifiers = [];
+          }
+
+          // If this item is in one of the selected categories, add the modifier
+          if (categories.includes(item.category)) {
+            if (!item.modifiers.includes(modifierName)) {
+              item.modifiers.push(modifierName);
+            }
+          }
+        });
+
+        // Update the modifier's appliesTo field to reflect the assignment
+        const modifier = modifiers.find(m => m.name === modifierName);
+        if (modifier) {
+          modifier.appliesTo = 'category';
+          modifier.categoryNames = categories; // Store all assigned categories
+        }
+      }
     }
 
     // Step 4: Items