From 05f1c4e38562bebb0b48908c15b044e75c100ae8 Mon Sep 17 00:00:00 2001 From: Matt-patto Date: Wed, 23 Sep 2026 12:34:33 +1000 Subject: [PATCH] feat(tools): store selector candidates for browser clicks Snapshot refs and click results previously kept a single CSS path per element, leaving replay nothing to fall back on when the DOM shifts. Each snapshot ref now also records an xpath and test-id selector, and the click action returns the resolved candidate set in its details so downstream traces have alternatives to retry with. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../tools/src/__tests__/browser-tool.test.ts | 32 ++++++++++- packages/tools/src/browser/browser-tool.ts | 53 ++++++++++++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/packages/tools/src/__tests__/browser-tool.test.ts b/packages/tools/src/__tests__/browser-tool.test.ts index 5d7511d..0eb49cb 100644 --- a/packages/tools/src/__tests__/browser-tool.test.ts +++ b/packages/tools/src/__tests__/browser-tool.test.ts @@ -63,6 +63,8 @@ function createManager() { role: "button", name: "Send", selector: "#send", + xpath: "//*[@id=\"send\"]", + testId: "[data-testid=\"send\"]", nth: 0, box: { x: 10, y: 20, width: 30, height: 12 }, }, @@ -154,7 +156,11 @@ describe("createBrowserTool", () => { const snapshot = await tool.execute("id", { action: "snapshot" }); expect((snapshot.content[0] as any).text).toContain("[1] button \"Send\""); - expect((snapshot.details as any).refs["1"]).toMatchObject({ selector: "#send" }); + expect((snapshot.details as any).refs["1"]).toMatchObject({ + selector: "#send", + xpath: "//*[@id=\"send\"]", + testId: "[data-testid=\"send\"]", + }); const ariaSnapshot = await tool.execute("id", { action: "snapshot", format: "aria" }); expect((ariaSnapshot.content[0] as any).text).toContain("ARIA snapshot"); @@ -168,6 +174,30 @@ describe("createBrowserTool", () => { expect(manager.close).toHaveBeenCalled(); }); + it("stores selector candidates for each click target", async () => { + const { manager } = createManager(); + const tool = createBrowserTool(manager as any); + + await tool.execute("id", { action: "snapshot" }); + const refClick = await tool.execute("id", { action: "click", ref: "1" }); + expect(refClick.details).toMatchObject({ + action: "click", + targetId: "tab_1", + target: "ref 1 (Send)", + selectors: { + css: "#send", + xpath: "//*[@id=\"send\"]", + testId: "[data-testid=\"send\"]", + }, + }); + + const directClick = await tool.execute("id", { action: "click", selector: "button" }); + expect(directClick.details).toMatchObject({ + action: "click", + selectors: { css: "button" }, + }); + }); + it("does not crash when evaluate resolves to undefined", async () => { const { manager, page } = createManager(); (page.evaluate as any).mockResolvedValueOnce(undefined); diff --git a/packages/tools/src/browser/browser-tool.ts b/packages/tools/src/browser/browser-tool.ts index 7bd1685..eff58de 100644 --- a/packages/tools/src/browser/browser-tool.ts +++ b/packages/tools/src/browser/browser-tool.ts @@ -101,6 +101,8 @@ interface BrowserSnapshotRef { value?: string; description?: string; selector: string; + xpath?: string; + testId?: string; nth: number; box?: { x: number; y: number; width: number; height: number }; } @@ -557,7 +559,7 @@ export function createBrowserTool( element?: string; required?: boolean; label?: string; - }): { locator: any; description: string } | undefined => { + }): { locator: any; description: string; selectors: Record } | undefined => { const refValue = params.ref?.trim(); if (refValue) { const snapRef = getSnapshotRef(page, refValue); @@ -565,6 +567,11 @@ export function createBrowserTool( return { locator: resolveLocatorFromSelector(page, snapRef.selector), description: `ref ${refValue}${snapRef.name ? ` (${snapRef.name})` : ""}`, + selectors: { + css: snapRef.selector, + ...(snapRef.xpath ? { xpath: snapRef.xpath } : {}), + ...(snapRef.testId ? { testId: snapRef.testId } : {}), + }, }; } throw new Error(`Unknown snapshot ref: ${refValue}. Take a new snapshot or pass selector explicitly.`); @@ -575,6 +582,7 @@ export function createBrowserTool( return { locator: resolveLocatorFromSelector(page, selector), description: `selector ${selector}`, + selectors: { css: selector }, }; } @@ -802,6 +810,38 @@ export function createBrowserTool( return `body > ${parts.join(" > ")}`; }; + const xpathPath = (element: BrowserDomElementLike): string => { + if (element.id) { + return `//*[@id="${element.id.replace(/"/g, '\\"')}"]`; + } + const parts: string[] = []; + let current: BrowserDomElementLike | null | undefined = element; + while (current) { + const tag = current.tagName.toLowerCase(); + const parent: BrowserDomElementLike | null | undefined = current.parentElement; + if (!parent || tag === "html") { + parts.unshift(tag); + break; + } + const siblings = Array.from(parent.children as ArrayLike) + .filter((child) => child.tagName === current?.tagName); + const index = siblings.indexOf(current) + 1; + parts.unshift(siblings.length > 1 ? `${tag}[${Math.max(1, index)}]` : tag); + current = parent; + } + return `/${parts.join("/")}`; + }; + + const testIdSelector = (element: BrowserDomElementLike): string | undefined => { + for (const attr of ["data-testid", "data-test-id", "data-test", "data-cy", "data-qa"]) { + const value = element.getAttribute(attr)?.trim(); + if (value) { + return `[${attr}="${value.replace(/"/g, '\\"')}"]`; + } + } + return undefined; + }; + const dedup = new Set(); const elements = Array.from(root.querySelectorAll(candidateSelector)) .filter((element) => visible(element)) @@ -832,6 +872,8 @@ export function createBrowserTool( value: value || undefined, description, selector: cssPath(element), + xpath: xpathPath(element), + testId: testIdSelector(element), nth, box: { x: Math.max(0, Math.round(rect.left)), @@ -1256,6 +1298,8 @@ export function createBrowserTool( name: node.name, nth: node.nth, selector: node.selector, + xpath: node.xpath, + testId: node.testId, }, ])), stats: snapshot.stats, @@ -1315,7 +1359,12 @@ export function createBrowserTool( label: "selector or ref", }); const verb = await clickLocator(locatorInfo!.locator, params); - return textResult(`${capitalize(verb)}: ${locatorInfo!.description}`); + return textResult(`${capitalize(verb)}: ${locatorInfo!.description}`, { + action: "click", + targetId: activeBrowserManager.getTabId(page), + target: locatorInfo!.description, + selectors: locatorInfo!.selectors, + }); } case "type": {