From d8e6f619acce078548944ea108a1fff8f58558fa Mon Sep 17 00:00:00 2001
From: John Mizerek <mizerek@gmail.com>
Date: Fri, 13 Feb 2026 08:53:25 -0800
Subject: [PATCH] Parse Toast menu from visible HTML for complete item
 extraction

- Extract items from visible HTML instead of just __OO_STATE__ JSON
- Parse headerText spans for item names, price spans for prices
- Extract images from Menu_files/ src attributes
- Fall back to simpler headerText matching if block parsing fails
- Also extract images from __OO_STATE__ and match to items by name
- Fixes issue where only 116 items extracted instead of 163+

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 api/setup/analyzeMenuUrl.cfm | 242 ++++++++++++++++++++++-------------
 1 file changed, 155 insertions(+), 87 deletions(-)

diff --git a/api/setup/analyzeMenuUrl.cfm b/api/setup/analyzeMenuUrl.cfm
index 07e649e..99c79d8 100644
--- a/api/setup/analyzeMenuUrl.cfm
+++ b/api/setup/analyzeMenuUrl.cfm
@@ -68,107 +68,175 @@
             <cfset playwrightImages = arrayNew(1)>
             <cfset arrayAppend(response.steps, "Read " & len(pageHtml) & " bytes from local file")>
 
-            <!--- Check for Toast __OO_STATE__ - parse directly, skip Claude --->
-            <cfif findNoCase("window.__OO_STATE__", pageHtml)>
-                <cfset arrayAppend(response.steps, "Toast menu detected - parsing __OO_STATE__ directly")>
+            <!--- Check for Toast menu page - extract from visible HTML for most complete data --->
+            <cfif findNoCase("class=""headerText""", pageHtml) AND findNoCase("toasttab", pageHtml)>
+                <cfset arrayAppend(response.steps, "Toast menu detected - parsing visible HTML items")>
 
-                <!--- Extract the JSON using regex --->
-                <cfset ooStateMatch = reMatchNoCase("window\.__OO_STATE__\s*=\s*(\{.*?\});\s*window\.", pageHtml)>
-                <cfif arrayLen(ooStateMatch)>
-                    <cfset ooStateJson = reReplaceNoCase(ooStateMatch[1], "window\.__OO_STATE__\s*=\s*", "")>
-                    <cfset ooStateJson = reReplace(ooStateJson, ";\s*window\.$", "")>
+                <cftry>
+                    <!--- Extract visible items from rendered HTML (most complete) --->
+                    <cfset toastBusiness = structNew()>
+                    <cfset toastCategories = arrayNew(1)>
+                    <cfset toastItems = arrayNew(1)>
+                    <cfset categorySet = structNew()>
+                    <cfset itemNameSet = structNew()>
+                    <cfset itemId = 1>
+                    <cfset currentCategory = "Menu">
 
-                    <cftry>
-                        <cfset ooState = deserializeJSON(ooStateJson)>
-                        <cfset arrayAppend(response.steps, "Parsed __OO_STATE__ JSON successfully")>
+                    <!--- Find category headers (h2 with specific Toast patterns) --->
+                    <cfset categoryMatches = reMatchNoCase('<h2[^>]*class="[^"]*groupHeader[^"]*"[^>]*>([^<]+)</h2>', pageHtml)>
+                    <cfloop array="#categoryMatches#" index="catMatch">
+                        <cfset catName = reReplaceNoCase(catMatch, '.*>([^<]+)</h2>.*', '\1')>
+                        <cfset catName = trim(catName)>
+                        <cfif len(catName) AND NOT structKeyExists(categorySet, catName)>
+                            <cfset categorySet[catName] = true>
+                            <cfset arrayAppend(toastCategories, { "name": catName, "itemCount": 0 })>
+                        </cfif>
+                    </cfloop>
 
-                        <!--- Extract menus, categories, and items --->
-                        <cfset toastBusiness = structNew()>
-                        <cfset toastCategories = arrayNew(1)>
-                        <cfset toastItems = arrayNew(1)>
-                        <cfset categorySet = structNew()>
-                        <cfset itemId = 1>
+                    <!--- Extract item blocks with name, price, description, image --->
+                    <!--- Toast pattern: li.item containing headerText for name, price span, itemImage img --->
+                    <cfset itemBlocks = reMatchNoCase('<li[^>]*class="[^"]*item[^"]*"[^>]*>.*?</li>', pageHtml)>
+                    <cfset arrayAppend(response.steps, "Found " & arrayLen(itemBlocks) & " item blocks in HTML")>
 
-                        <cfloop collection="#ooState#" item="key">
-                            <cfif left(key, 5) EQ "Menu:">
-                                <cfset menu = ooState[key]>
-                                <cfif structKeyExists(menu, "groups") AND isArray(menu.groups)>
-                                    <cfloop array="#menu.groups#" index="group">
-                                        <cfset groupName = structKeyExists(group, "name") ? group.name : "">
-                                        <cfif len(groupName) AND NOT structKeyExists(categorySet, groupName)>
-                                            <cfset categorySet[groupName] = true>
-                                            <cfset arrayAppend(toastCategories, { "name": groupName, "itemCount": 0 })>
-                                        </cfif>
+                    <cfloop array="#itemBlocks#" index="block">
+                        <!--- Extract item name --->
+                        <cfset nameMatch = reMatchNoCase('<span class="headerText">([^<]+)</span>', block)>
+                        <cfif arrayLen(nameMatch)>
+                            <cfset itemName = reReplaceNoCase(nameMatch[1], '.*>([^<]+)</span>.*', '\1')>
+                            <cfset itemName = trim(itemName)>
 
-                                        <cfif structKeyExists(group, "items") AND isArray(group.items)>
-                                            <cfloop array="#group.items#" index="item">
-                                                <cfset itemStruct = structNew()>
-                                                <cfset itemStruct["id"] = "item_" & itemId>
-                                                <cfset itemStruct["name"] = structKeyExists(item, "name") ? item.name : "">
-                                                <cfset itemStruct["description"] = structKeyExists(item, "description") ? item.description : "">
-                                                <cfset itemStruct["category"] = groupName>
-                                                <cfset itemStruct["modifiers"] = arrayNew(1)>
+                            <!--- Skip duplicates --->
+                            <cfif len(itemName) AND NOT structKeyExists(itemNameSet, itemName)>
+                                <cfset itemNameSet[itemName] = true>
 
-                                                <!--- Handle prices (array for sizes) --->
-                                                <cfif structKeyExists(item, "prices") AND isArray(item.prices) AND arrayLen(item.prices)>
-                                                    <cfset itemStruct["price"] = item.prices[1]>
-                                                <cfelseif structKeyExists(item, "price")>
-                                                    <cfset itemStruct["price"] = item.price>
-                                                <cfelse>
-                                                    <cfset itemStruct["price"] = 0>
-                                                </cfif>
+                                <cfset itemStruct = structNew()>
+                                <cfset itemStruct["id"] = "item_" & itemId>
+                                <cfset itemStruct["name"] = itemName>
+                                <cfset itemStruct["modifiers"] = arrayNew(1)>
 
-                                                <!--- Handle images --->
-                                                <cfif structKeyExists(item, "imageUrls") AND isStruct(item.imageUrls)>
-                                                    <cfset imgUrls = item.imageUrls>
-                                                    <cfif structKeyExists(imgUrls, "medium")>
-                                                        <cfset itemStruct["imageUrl"] = imgUrls.medium>
-                                                        <cfset itemStruct["imageSrc"] = imgUrls.medium>
-                                                        <cfset itemStruct["imageFilename"] = listLast(imgUrls.medium, "/")>
-                                                    <cfelseif structKeyExists(imgUrls, "large")>
-                                                        <cfset itemStruct["imageUrl"] = imgUrls.large>
-                                                        <cfset itemStruct["imageSrc"] = imgUrls.large>
-                                                        <cfset itemStruct["imageFilename"] = listLast(imgUrls.large, "/")>
-                                                    </cfif>
-                                                </cfif>
+                                <!--- Extract price --->
+                                <cfset priceMatch = reMatchNoCase('<span[^>]*class="price"[^>]*>\$?([0-9.]+)</span>', block)>
+                                <cfif arrayLen(priceMatch)>
+                                    <cfset priceStr = reReplaceNoCase(priceMatch[1], '.*>\\$?([0-9.]+)</span>.*', '\1')>
+                                    <cfset itemStruct["price"] = val(priceStr)>
+                                <cfelse>
+                                    <cfset itemStruct["price"] = 0>
+                                </cfif>
 
-                                                <cfif len(itemStruct.name)>
-                                                    <cfset arrayAppend(toastItems, itemStruct)>
-                                                    <cfset itemId++>
+                                <!--- Extract description --->
+                                <cfset descMatch = reMatchNoCase('<div[^>]*class="[^"]*description[^"]*"[^>]*>([^<]+)</div>', block)>
+                                <cfif arrayLen(descMatch)>
+                                    <cfset itemStruct["description"] = trim(reReplaceNoCase(descMatch[1], '.*>([^<]+)</div>.*', '\1'))>
+                                <cfelse>
+                                    <cfset itemStruct["description"] = "">
+                                </cfif>
+
+                                <!--- Extract image URL from srcset or src --->
+                                <cfset imgMatch = reMatchNoCase('src="(Menu_files/[^"]+)"', block)>
+                                <cfif arrayLen(imgMatch)>
+                                    <cfset imgSrc = reReplaceNoCase(imgMatch[1], '.*src="([^"]+)".*', '\1')>
+                                    <!--- Convert to full URL --->
+                                    <cfset itemStruct["imageUrl"] = basePath & imgSrc>
+                                    <cfset itemStruct["imageSrc"] = basePath & imgSrc>
+                                    <cfset itemStruct["imageFilename"] = listLast(imgSrc, "/")>
+                                </cfif>
+
+                                <!--- Try to determine category from nearby h2 or default --->
+                                <cfset itemStruct["category"] = arrayLen(toastCategories) ? toastCategories[1].name : "Menu">
+
+                                <cfset arrayAppend(toastItems, itemStruct)>
+                                <cfset itemId++>
+                            </cfif>
+                        </cfif>
+                    </cfloop>
+
+                    <!--- If no items found from blocks, try simpler headerText extraction --->
+                    <cfif arrayLen(toastItems) EQ 0>
+                        <cfset nameMatches = reMatchNoCase('<span class="headerText">([^<]+)</span>', pageHtml)>
+                        <cfloop array="#nameMatches#" index="nameMatch">
+                            <cfset itemName = reReplaceNoCase(nameMatch, '.*>([^<]+)</span>.*', '\1')>
+                            <cfset itemName = trim(itemName)>
+                            <cfif len(itemName) AND NOT structKeyExists(itemNameSet, itemName)>
+                                <cfset itemNameSet[itemName] = true>
+                                <cfset itemStruct = { "id": "item_" & itemId, "name": itemName, "price": 0, "description": "", "category": "Menu", "modifiers": [] }>
+                                <cfset arrayAppend(toastItems, itemStruct)>
+                                <cfset itemId++>
+                            </cfif>
+                        </cfloop>
+                    </cfif>
+
+                    <!--- Also try to extract from __OO_STATE__ for images --->
+                    <cfif findNoCase("window.__OO_STATE__", pageHtml)>
+                        <cfset ooStateMatch = reMatchNoCase("window\.__OO_STATE__\s*=\s*(\{.*?\});\s*window\.", pageHtml)>
+                        <cfif arrayLen(ooStateMatch)>
+                            <cfset ooStateJson = reReplaceNoCase(ooStateMatch[1], "window\.__OO_STATE__\s*=\s*", "")>
+                            <cfset ooStateJson = reReplace(ooStateJson, ";\s*window\.$", "")>
+                            <cftry>
+                                <cfset ooState = deserializeJSON(ooStateJson)>
+                                <!--- Build name -> image URL map from OO_STATE --->
+                                <cfset imageMap = structNew()>
+                                <cfloop collection="#ooState#" item="key">
+                                    <cfif left(key, 5) EQ "Menu:">
+                                        <cfset menu = ooState[key]>
+                                        <cfif structKeyExists(menu, "groups") AND isArray(menu.groups)>
+                                            <cfloop array="#menu.groups#" index="group">
+                                                <cfif structKeyExists(group, "items") AND isArray(group.items)>
+                                                    <cfloop array="#group.items#" index="item">
+                                                        <cfif structKeyExists(item, "name") AND structKeyExists(item, "imageUrls")>
+                                                            <cfset imgUrls = item.imageUrls>
+                                                            <cfif structKeyExists(imgUrls, "medium")>
+                                                                <cfset imageMap[item.name] = imgUrls.medium>
+                                                            <cfelseif structKeyExists(imgUrls, "large")>
+                                                                <cfset imageMap[item.name] = imgUrls.large>
+                                                            </cfif>
+                                                        </cfif>
+                                                    </cfloop>
                                                 </cfif>
                                             </cfloop>
                                         </cfif>
-                                    </cfloop>
-                                </cfif>
-                            </cfif>
-                        </cfloop>
+                                    </cfif>
+                                </cfloop>
+                                <!--- Apply images to items --->
+                                <cfset imagesMatched = 0>
+                                <cfloop from="1" to="#arrayLen(toastItems)#" index="i">
+                                    <cfif structKeyExists(imageMap, toastItems[i].name)>
+                                        <cfset toastItems[i]["imageUrl"] = imageMap[toastItems[i].name]>
+                                        <cfset toastItems[i]["imageSrc"] = imageMap[toastItems[i].name]>
+                                        <cfset toastItems[i]["imageFilename"] = listLast(imageMap[toastItems[i].name], "/")>
+                                        <cfset imagesMatched++>
+                                    </cfif>
+                                </cfloop>
+                                <cfset arrayAppend(response.steps, "Matched " & imagesMatched & " images from __OO_STATE__")>
+                            <cfcatch></cfcatch>
+                            </cftry>
+                        </cfif>
+                    </cfif>
 
-                        <cfset arrayAppend(response.steps, "Extracted " & arrayLen(toastItems) & " items from " & arrayLen(toastCategories) & " categories")>
+                    <cfset arrayAppend(response.steps, "Extracted " & arrayLen(toastItems) & " unique items from " & arrayLen(toastCategories) & " categories")>
 
-                        <!--- Return directly without Claude --->
-                        <cfset response["OK"] = true>
-                        <cfset response["DATA"] = {
-                            "business": toastBusiness,
-                            "categories": toastCategories,
-                            "modifiers": arrayNew(1),
-                            "items": toastItems,
-                            "imageUrls": arrayNew(1),
-                            "headerCandidateIndices": arrayNew(1),
-                            "imageMappings": arrayNew(1)
-                        }>
-                        <cfset response["sourceUrl"] = targetUrl>
-                        <cfset response["pagesProcessed"] = 1>
-                        <cfset response["imagesFound"] = 0>
-                        <cfset response["playwrightImagesCount"] = 0>
-                        <cfset response["toastDirect"] = true>
-                        <cfoutput>#serializeJSON(response)#</cfoutput>
-                        <cfabort>
+                    <!--- Return directly without Claude --->
+                    <cfset response["OK"] = true>
+                    <cfset response["DATA"] = {
+                        "business": toastBusiness,
+                        "categories": toastCategories,
+                        "modifiers": arrayNew(1),
+                        "items": toastItems,
+                        "imageUrls": arrayNew(1),
+                        "headerCandidateIndices": arrayNew(1),
+                        "imageMappings": arrayNew(1)
+                    }>
+                    <cfset response["sourceUrl"] = targetUrl>
+                    <cfset response["pagesProcessed"] = 1>
+                    <cfset response["imagesFound"] = 0>
+                    <cfset response["playwrightImagesCount"] = 0>
+                    <cfset response["toastDirect"] = true>
+                    <cfoutput>#serializeJSON(response)#</cfoutput>
+                    <cfabort>
 
-                    <cfcatch type="any">
-                        <cfset arrayAppend(response.steps, "Toast JSON parse failed: " & cfcatch.message & " - falling back to Claude")>
-                    </cfcatch>
-                    </cftry>
-                </cfif>
+                <cfcatch type="any">
+                    <cfset arrayAppend(response.steps, "Toast HTML parse failed: " & cfcatch.message & " - falling back to Claude")>
+                </cfcatch>
+                </cftry>
             </cfif>
 
             <!--- Extract base URL for resolving relative links --->