Improve DoorDash modifier extraction: pass item names to Playwright
- Pass extracted item names via temp JSON file so Playwright knows exactly what to click instead of guessing from DOM selectors (7 → 171 items) - Use TreeWalker for exact text matching and aggressive scrolling - Better price parsing: handle cents (int), dollars (string), displayPrice - Improved modal dismissal with overlay click fallback Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
John Mizerek
parent
b14f26ed47
commit
f5974a5fa2
2 changed files with 143 additions and 108 deletions
|
|
@ -1442,12 +1442,22 @@
|
|||
<cfset ddItemModMap = structNew()>
|
||||
<cftry>
|
||||
<cfset arrayAppend(response.steps, "Running stealth Playwright for modifier extraction...")>
|
||||
<!--- Write item names to temp file so Playwright knows what to click --->
|
||||
<cfset ddItemNames = arrayNew(1)>
|
||||
<cfloop array="#ddItems#" index="ddi">
|
||||
<cfset arrayAppend(ddItemNames, ddi.name)>
|
||||
</cfloop>
|
||||
<cfset ddTempFile = "/tmp/dd-items-#createUUID()#.json">
|
||||
<cffile action="write" file="#ddTempFile#" output="#serializeJSON(ddItemNames)#" charset="utf-8">
|
||||
|
||||
<cfset modTimeout = 180000 + (arrayLen(ddItems) * 1500)>
|
||||
<cfif modTimeout GT 600000><cfset modTimeout = 600000></cfif>
|
||||
<cfexecute name="/opt/playwright/run-doordash-modifiers.sh"
|
||||
arguments="#targetUrl#"
|
||||
arguments="#targetUrl# #ddTempFile#"
|
||||
timeout="#int(modTimeout / 1000)#"
|
||||
variable="ddModResult" />
|
||||
<!--- Clean up temp file --->
|
||||
<cftry><cffile action="delete" file="#ddTempFile#"><cfcatch></cfcatch></cftry>
|
||||
|
||||
<cfif len(trim(ddModResult))>
|
||||
<cfset ddModData = deserializeJSON(trim(ddModResult))>
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
const { chromium } = require("playwright-extra");
|
||||
const stealth = require("puppeteer-extra-plugin-stealth");
|
||||
const fs = require("fs");
|
||||
chromium.use(stealth());
|
||||
|
||||
(async () => {
|
||||
const url = process.argv[2];
|
||||
const itemNamesFile = process.argv[3]; // Optional: JSON file with array of item names
|
||||
if (!url) {
|
||||
console.log(JSON.stringify({ error: "URL required", modifiers: [], itemModifierMap: {} }));
|
||||
process.exit(1);
|
||||
|
|
@ -11,6 +13,17 @@ chromium.use(stealth());
|
|||
|
||||
const log = (msg) => process.stderr.write("[dd-mod] " + msg + "\n");
|
||||
|
||||
// Load item names if provided (from CFML fast-path)
|
||||
let knownItemNames = [];
|
||||
if (itemNamesFile && fs.existsSync(itemNamesFile)) {
|
||||
try {
|
||||
knownItemNames = JSON.parse(fs.readFileSync(itemNamesFile, "utf8"));
|
||||
log("Loaded " + knownItemNames.length + " item names from file");
|
||||
} catch (e) {
|
||||
log("Failed to load item names file: " + e.message);
|
||||
}
|
||||
}
|
||||
|
||||
let browser;
|
||||
try {
|
||||
browser = await chromium.launch({ headless: true });
|
||||
|
|
@ -31,12 +44,10 @@ chromium.use(stealth());
|
|||
const ct = response.headers()["content-type"] || "";
|
||||
if (ct.includes("json")) {
|
||||
const body = await response.json();
|
||||
// DoorDash itemPage response structure
|
||||
if (body && body.data && body.data.itemPage) {
|
||||
latestItemPage = body.data.itemPage;
|
||||
responseCount++;
|
||||
}
|
||||
// Some DoorDash endpoints wrap in array
|
||||
if (Array.isArray(body)) {
|
||||
for (const entry of body) {
|
||||
if (entry && entry.data && entry.data.itemPage) {
|
||||
|
|
@ -54,49 +65,75 @@ chromium.use(stealth());
|
|||
await page.goto(url, { waitUntil: "load", timeout: 60000 });
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
// Scroll to load all items
|
||||
const scrollHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
const viewportHeight = await page.evaluate(() => window.innerHeight);
|
||||
const scrollSteps = Math.min(Math.ceil(scrollHeight / viewportHeight), 20);
|
||||
// Aggressive scroll to force DoorDash to render all items
|
||||
log("Scrolling to load all items...");
|
||||
let lastHeight = 0;
|
||||
for (let round = 0; round < 3; round++) {
|
||||
const scrollHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
const viewportHeight = await page.evaluate(() => window.innerHeight);
|
||||
const scrollSteps = Math.ceil(scrollHeight / viewportHeight);
|
||||
|
||||
for (let i = 0; i < scrollSteps; i++) {
|
||||
await page.evaluate((step) => {
|
||||
window.scrollTo(0, step * window.innerHeight);
|
||||
}, i + 1);
|
||||
await page.waitForTimeout(300);
|
||||
for (let i = 0; i <= scrollSteps; i++) {
|
||||
await page.evaluate((y) => window.scrollTo(0, y), i * viewportHeight);
|
||||
await page.waitForTimeout(250);
|
||||
}
|
||||
await page.waitForTimeout(500);
|
||||
|
||||
const newHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
if (newHeight === lastHeight) break;
|
||||
lastHeight = newHeight;
|
||||
}
|
||||
await page.evaluate(() => window.scrollTo(0, 0));
|
||||
await page.waitForTimeout(1000);
|
||||
await page.waitForTimeout(500);
|
||||
|
||||
// Find all clickable menu item elements
|
||||
// DoorDash renders items as buttons/anchors with item names and images
|
||||
const itemElements = await page.evaluate(() => {
|
||||
const items = [];
|
||||
const seen = new Set();
|
||||
// If we have known item names, use them to find and click elements
|
||||
// Otherwise fall back to DOM auto-detection
|
||||
let itemsToClick = [];
|
||||
|
||||
// Strategy 1: Look for item cards with data-anchor-id containing "MenuItem"
|
||||
document.querySelectorAll('[data-anchor-id*="MenuItem"]').forEach(el => {
|
||||
const nameEl = el.querySelector('[data-telemetry-id="storeMenuItem.title"]') ||
|
||||
el.querySelector('span[class*="Text"]') ||
|
||||
el.querySelector('h3') ||
|
||||
el.querySelector('span');
|
||||
if (nameEl) {
|
||||
const name = nameEl.textContent.trim();
|
||||
if (name && !seen.has(name) && name.length > 1 && name.length < 200) {
|
||||
seen.add(name);
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width > 0 && rect.height > 0) {
|
||||
items.push({ name, x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 });
|
||||
if (knownItemNames.length > 0) {
|
||||
// Find each known item in the DOM by text content
|
||||
log("Searching DOM for " + knownItemNames.length + " known items...");
|
||||
itemsToClick = await page.evaluate((names) => {
|
||||
const found = [];
|
||||
const allElements = document.querySelectorAll('span, h3, h4, p, div');
|
||||
|
||||
// Build a map of text -> element for fast lookup
|
||||
const textMap = new Map();
|
||||
for (const el of allElements) {
|
||||
// Only use leaf-ish elements (avoid containers that contain the whole menu)
|
||||
if (el.children.length > 5) continue;
|
||||
const text = el.textContent.trim();
|
||||
if (text.length > 1 && text.length < 200 && !textMap.has(text)) {
|
||||
textMap.set(text, el);
|
||||
}
|
||||
}
|
||||
|
||||
for (const name of names) {
|
||||
const el = textMap.get(name);
|
||||
if (el) {
|
||||
// Find the clickable parent (the item card)
|
||||
const clickable = el.closest('a, button, [role="button"], [tabindex="0"], [data-anchor-id]')
|
||||
|| el.parentElement?.closest('a, button, [role="button"], [tabindex="0"], [data-anchor-id]')
|
||||
|| el.parentElement;
|
||||
if (clickable) {
|
||||
const rect = clickable.getBoundingClientRect();
|
||||
if (rect.width > 0 && rect.height > 0) {
|
||||
found.push({ name, y: rect.y + window.scrollY });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return found;
|
||||
}, knownItemNames);
|
||||
|
||||
// Strategy 2: Look for buttons/divs containing item names with prices
|
||||
if (items.length === 0) {
|
||||
document.querySelectorAll('button, [role="button"], [data-testid*="item"], [data-testid*="menu"]').forEach(el => {
|
||||
log("Found " + itemsToClick.length + "/" + knownItemNames.length + " items in DOM");
|
||||
} else {
|
||||
// Auto-detect from DOM (fallback)
|
||||
itemsToClick = await page.evaluate(() => {
|
||||
const items = [];
|
||||
const seen = new Set();
|
||||
document.querySelectorAll('[data-anchor-id*="MenuItem"], button, [role="button"]').forEach(el => {
|
||||
const text = el.textContent || "";
|
||||
// Items typically have a price like $X.XX
|
||||
if (text.match(/\$\d+\.\d{2}/) && text.length < 500) {
|
||||
const lines = text.split("\n").map(l => l.trim()).filter(l => l.length > 0);
|
||||
const name = lines[0];
|
||||
|
|
@ -104,88 +141,66 @@ chromium.use(stealth());
|
|||
seen.add(name);
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width > 0 && rect.height > 0) {
|
||||
items.push({ name, x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 });
|
||||
items.push({ name, y: rect.y + window.scrollY });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return items;
|
||||
});
|
||||
log("Auto-detected " + itemsToClick.length + " clickable items");
|
||||
}
|
||||
|
||||
// Strategy 3: Generic - find any clickable element with an image and text nearby
|
||||
if (items.length === 0) {
|
||||
document.querySelectorAll('img[src*="cdn4dd"]').forEach(img => {
|
||||
const parent = img.closest('a, button, [role="button"], [tabindex="0"]') || img.parentElement.parentElement;
|
||||
if (parent) {
|
||||
const nameEl = parent.querySelector('span, h3, h4, p');
|
||||
if (nameEl) {
|
||||
const name = nameEl.textContent.trim();
|
||||
if (name && !seen.has(name) && name.length > 1 && name.length < 200) {
|
||||
seen.add(name);
|
||||
const rect = parent.getBoundingClientRect();
|
||||
if (rect.width > 0 && rect.height > 0) {
|
||||
items.push({ name, x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return items;
|
||||
});
|
||||
|
||||
log("Found " + itemElements.length + " clickable items on page");
|
||||
|
||||
if (itemElements.length === 0) {
|
||||
log("No clickable items found, trying fallback...");
|
||||
// Take a screenshot for debugging
|
||||
if (itemsToClick.length === 0) {
|
||||
console.log(JSON.stringify({ error: "No clickable items found", modifiers: [], itemModifierMap: {} }));
|
||||
await browser.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Click each item, capture modifier data
|
||||
const allModifierGroups = new Map(); // name -> modifier group data
|
||||
const itemModifierMap = {}; // item name -> [modifier group names]
|
||||
const allModifierGroups = new Map();
|
||||
const itemModifierMap = {};
|
||||
let clickedCount = 0;
|
||||
let modItemCount = 0;
|
||||
let noModCount = 0;
|
||||
|
||||
// Limit to prevent timeouts (DoorDash has many items)
|
||||
const maxClicks = Math.min(itemElements.length, 200);
|
||||
const maxClicks = Math.min(itemsToClick.length, 250);
|
||||
|
||||
for (let i = 0; i < maxClicks; i++) {
|
||||
const item = itemElements[i];
|
||||
const item = itemsToClick[i];
|
||||
try {
|
||||
latestItemPage = null;
|
||||
|
||||
// Scroll item into view and click by coordinates
|
||||
await page.evaluate((y) => window.scrollTo(0, y - 300), item.y);
|
||||
// Scroll to item position
|
||||
await page.evaluate((y) => window.scrollTo(0, Math.max(0, y - 300)), item.y);
|
||||
await page.waitForTimeout(200);
|
||||
|
||||
// Recalculate position after scroll
|
||||
const freshPos = await page.evaluate((itemName) => {
|
||||
const els = document.querySelectorAll('[data-anchor-id*="MenuItem"], button, [role="button"]');
|
||||
for (const el of els) {
|
||||
if (el.textContent.includes(itemName)) {
|
||||
const rect = el.getBoundingClientRect();
|
||||
// Find the item element by name and click it
|
||||
const clicked = await page.evaluate((itemName) => {
|
||||
// Find the text node with this exact name
|
||||
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
|
||||
while (walker.nextNode()) {
|
||||
if (walker.currentNode.textContent.trim() === itemName) {
|
||||
const el = walker.currentNode.parentElement;
|
||||
const clickable = el.closest('a, button, [role="button"], [tabindex="0"], [data-anchor-id]')
|
||||
|| el.parentElement?.closest('a, button, [role="button"], [tabindex="0"], [data-anchor-id]')
|
||||
|| el;
|
||||
const rect = clickable.getBoundingClientRect();
|
||||
if (rect.width > 0 && rect.height > 0) {
|
||||
return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2, found: true };
|
||||
clickable.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { found: false };
|
||||
return false;
|
||||
}, item.name);
|
||||
|
||||
if (!freshPos.found) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await page.mouse.click(freshPos.x, freshPos.y);
|
||||
if (!clicked) continue;
|
||||
clickedCount++;
|
||||
|
||||
// Wait for GraphQL response (up to 4s)
|
||||
// Wait for GraphQL response (up to 5s)
|
||||
const startTime = Date.now();
|
||||
while (!latestItemPage && Date.now() - startTime < 4000) {
|
||||
while (!latestItemPage && Date.now() - startTime < 5000) {
|
||||
await page.waitForTimeout(150);
|
||||
}
|
||||
|
||||
|
|
@ -199,11 +214,19 @@ chromium.use(stealth());
|
|||
const options = [];
|
||||
if (ol.options && Array.isArray(ol.options)) {
|
||||
for (const opt of ol.options) {
|
||||
const price = opt.price ? (typeof opt.price === "number" ? opt.price / 100 : parseFloat(opt.price) || 0) : 0;
|
||||
options.push({
|
||||
name: opt.name || "",
|
||||
price: price
|
||||
});
|
||||
// DoorDash prices: sometimes cents (int), sometimes dollars (string like "$1.50")
|
||||
let price = 0;
|
||||
if (opt.price) {
|
||||
if (typeof opt.price === "number") {
|
||||
price = opt.price > 100 ? opt.price / 100 : opt.price;
|
||||
} else if (typeof opt.price === "string") {
|
||||
price = parseFloat(opt.price.replace(/[^0-9.]/g, "")) || 0;
|
||||
}
|
||||
}
|
||||
if (opt.displayPrice) {
|
||||
price = parseFloat(String(opt.displayPrice).replace(/[^0-9.]/g, "")) || price;
|
||||
}
|
||||
options.push({ name: opt.name || "", price: price });
|
||||
}
|
||||
}
|
||||
allModifierGroups.set(olName, {
|
||||
|
|
@ -218,22 +241,24 @@ chromium.use(stealth());
|
|||
}
|
||||
itemModifierMap[item.name] = modNames;
|
||||
modItemCount++;
|
||||
} else {
|
||||
noModCount++;
|
||||
}
|
||||
} else {
|
||||
noModCount++;
|
||||
}
|
||||
|
||||
// Close modal (press Escape or click outside)
|
||||
// Close modal
|
||||
await page.keyboard.press("Escape");
|
||||
await page.waitForTimeout(400);
|
||||
await page.waitForTimeout(350);
|
||||
|
||||
// Check if modal is still open, click overlay if so
|
||||
const modalStillOpen = await page.evaluate(() => {
|
||||
const overlay = document.querySelector('[data-testid="modal-overlay"], [class*="ModalOverlay"], [class*="overlay"]');
|
||||
return !!overlay;
|
||||
// Double-check modal is closed
|
||||
const stillOpen = await page.evaluate(() => {
|
||||
const overlay = document.querySelector('[data-testid="modal-overlay"], [class*="ModalOverlay"], [class*="Overlay"]');
|
||||
if (overlay) { overlay.click(); return true; }
|
||||
return false;
|
||||
});
|
||||
if (modalStillOpen) {
|
||||
await page.mouse.click(10, 10);
|
||||
await page.waitForTimeout(300);
|
||||
}
|
||||
if (stillOpen) await page.waitForTimeout(300);
|
||||
|
||||
} catch (e) {
|
||||
log("Error clicking " + item.name + ": " + e.message);
|
||||
|
|
@ -241,13 +266,12 @@ chromium.use(stealth());
|
|||
await page.waitForTimeout(300);
|
||||
}
|
||||
|
||||
// Progress log every 20 items
|
||||
if ((i + 1) % 20 === 0) {
|
||||
log("Progress: " + (i + 1) + "/" + maxClicks + " clicked, " + modItemCount + " with modifiers");
|
||||
if ((i + 1) % 25 === 0) {
|
||||
log("Progress: " + (i + 1) + "/" + maxClicks + " | " + modItemCount + " with mods, " + noModCount + " without");
|
||||
}
|
||||
}
|
||||
|
||||
log("Done: " + clickedCount + " clicked, " + modItemCount + " items with modifiers, " + allModifierGroups.size + " unique modifier groups");
|
||||
log("Done: " + clickedCount + "/" + maxClicks + " clicked, " + modItemCount + " with modifiers, " + allModifierGroups.size + " unique groups");
|
||||
|
||||
const modifiers = Array.from(allModifierGroups.values());
|
||||
|
||||
|
|
@ -255,9 +279,10 @@ chromium.use(stealth());
|
|||
modifiers: modifiers,
|
||||
itemModifierMap: itemModifierMap,
|
||||
stats: {
|
||||
clickableItems: itemElements.length,
|
||||
totalItems: itemsToClick.length,
|
||||
clickedCount: clickedCount,
|
||||
itemsWithModifiers: modItemCount,
|
||||
itemsWithoutModifiers: noModCount,
|
||||
uniqueModifierGroups: modifiers.length
|
||||
}
|
||||
}));
|
||||
|
|
|
|||
Reference in a new issue