Skip to content

Commit e0d94c7

Browse files
WinifyWinify
committed
feat: Re-add get_elements tool for retrieving interactable page elements
- Added `get_elements` tool with configurable options like viewport filtering, container inclusion, and pagination. - Refactored `elements` resource to reuse `getElements` script for element retrieval logic. - Updated tests to ensure accurate behavior for `get_elements` functionality.
1 parent 59f83a5 commit e0d94c7

7 files changed

Lines changed: 265 additions & 74 deletions

File tree

‎src/resources/accessibility.resource.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ export async function readAccessibilityTree(params: {
1818
};
1919
}
2020

21-
const { limit = 100, offset = 0, roles } = params;
21+
const { limit = 0, offset = 0, roles } = params;
2222

2323
let nodes = await getBrowserAccessibilityTree(browser);
2424

@@ -68,7 +68,7 @@ export async function readAccessibilityTree(params: {
6868
export const accessibilityResource: ResourceDefinition = {
6969
name: 'session-current-accessibility',
7070
uri: 'wdio://session/current/accessibility',
71-
description: 'Accessibility tree for the current page',
71+
description: 'Accessibility tree for the current page. Returns all elements by default.',
7272
handler: async () => {
7373
const result = await readAccessibilityTree({});
7474
return { contents: [{ uri: 'wdio://session/current/accessibility', mimeType: result.mimeType, text: result.text }] };

‎src/resources/elements.resource.ts‎

Lines changed: 16 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,26 @@
11
import type { ResourceDefinition } from '../types/resource';
22
import { getBrowser } from '../session/state';
3-
import { getInteractableBrowserElements } from '../scripts/get-interactable-browser-elements';
4-
import { getMobileVisibleElements } from '../scripts/get-visible-mobile-elements';
3+
import { getElements } from '../scripts/get-elements';
54
import { encode } from '@toon-format/toon';
65

7-
async function readVisibleElements(params: {
8-
inViewportOnly?: boolean;
9-
includeContainers?: boolean;
10-
includeBounds?: boolean;
11-
limit?: number;
12-
offset?: number;
13-
}): Promise<{ mimeType: string; text: string }> {
14-
try {
15-
const browser = getBrowser();
16-
const {
17-
inViewportOnly = true,
18-
includeContainers = false,
19-
includeBounds = false,
20-
limit = 0,
21-
offset = 0,
22-
} = params;
23-
24-
let elements: { isInViewport?: boolean }[];
25-
26-
if (browser.isAndroid || browser.isIOS) {
27-
const platform = browser.isAndroid ? 'android' : 'ios';
28-
elements = await getMobileVisibleElements(browser, platform, { includeContainers, includeBounds });
29-
} else {
30-
elements = await getInteractableBrowserElements(browser, { includeBounds });
31-
}
32-
33-
if (inViewportOnly) {
34-
elements = elements.filter((el) => el.isInViewport !== false);
35-
}
36-
37-
const total = elements.length;
38-
39-
if (offset > 0) {
40-
elements = elements.slice(offset);
41-
}
42-
if (limit > 0) {
43-
elements = elements.slice(0, limit);
44-
}
45-
46-
const result: Record<string, unknown> = {
47-
total,
48-
showing: elements.length,
49-
hasMore: offset + elements.length < total,
50-
elements,
51-
};
52-
53-
const toon = encode(result).replace(/,""/g, ',').replace(/"",/g, ',');
54-
return { mimeType: 'text/plain', text: toon };
55-
} catch (e) {
56-
return { mimeType: 'text/plain', text: `Error getting visible elements: ${e}` };
57-
}
58-
}
59-
606
export const elementsResource: ResourceDefinition = {
617
name: 'session-current-elements',
628
uri: 'wdio://session/current/elements',
639
description: 'Interactable elements on the current page',
6410
handler: async () => {
65-
const result = await readVisibleElements({});
66-
return { contents: [{ uri: 'wdio://session/current/elements', mimeType: result.mimeType, text: result.text }] };
11+
try {
12+
const browser = getBrowser();
13+
const result = await getElements(browser, {});
14+
const text = encode(result).replace(/,""/g, ',').replace(/"",/g, ',');
15+
return { contents: [{ uri: 'wdio://session/current/elements', mimeType: 'text/plain', text }] };
16+
} catch (e) {
17+
return {
18+
contents: [{
19+
uri: 'wdio://session/current/elements',
20+
mimeType: 'text/plain',
21+
text: `Error getting visible elements: ${e}`
22+
}]
23+
};
24+
}
6725
},
68-
};
26+
};

‎src/scripts/get-elements.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import { getInteractableBrowserElements } from './get-interactable-browser-elements';
2+
import { getMobileVisibleElements } from './get-visible-mobile-elements';
3+
4+
export type VisibleElementsResult = {
5+
total: number;
6+
showing: number;
7+
hasMore: boolean;
8+
elements: unknown[];
9+
};
10+
11+
export async function getElements(
12+
browser: WebdriverIO.Browser,
13+
params: {
14+
inViewportOnly?: boolean;
15+
includeContainers?: boolean;
16+
includeBounds?: boolean;
17+
limit?: number;
18+
offset?: number;
19+
},
20+
): Promise<VisibleElementsResult> {
21+
const {
22+
inViewportOnly = true,
23+
includeContainers = false,
24+
includeBounds = false,
25+
limit = 0,
26+
offset = 0,
27+
} = params;
28+
29+
let elements: { isInViewport?: boolean }[];
30+
31+
if (browser.isAndroid || browser.isIOS) {
32+
const platform = browser.isAndroid ? 'android' : 'ios';
33+
elements = await getMobileVisibleElements(browser, platform, { includeContainers, includeBounds });
34+
} else {
35+
elements = await getInteractableBrowserElements(browser, { includeBounds });
36+
}
37+
38+
if (inViewportOnly) {
39+
elements = elements.filter((el) => el.isInViewport !== false);
40+
}
41+
42+
const total = elements.length;
43+
44+
if (offset > 0) {
45+
elements = elements.slice(offset);
46+
}
47+
if (limit > 0) {
48+
elements = elements.slice(0, limit);
49+
}
50+
51+
return {
52+
total,
53+
showing: elements.length,
54+
hasMore: offset + elements.length < total,
55+
elements,
56+
};
57+
}

‎src/server.ts‎

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#!/usr/bin/env node
22
import pkg from '../package.json' with { type: 'json' };
3+
import type { ToolCallback } from '@modelcontextprotocol/sdk/server/mcp';
34
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp';
45
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
56
import type { ToolDefinition } from './types/tool';
6-
import type { ToolCallback } from '@modelcontextprotocol/sdk/server/mcp';
77
import type { ResourceDefinition } from './types/resource';
88
import { navigateTool, navigateToolDefinition } from './tools/navigate.tool';
99
import { clickTool, clickToolDefinition } from './tools/click.tool';
@@ -34,25 +34,32 @@ import {
3434
} from './tools/device.tool';
3535
import { executeScriptTool, executeScriptToolDefinition } from './tools/execute-script.tool';
3636
import { executeSequenceTool, executeSequenceToolDefinition } from './tools/execute-sequence.tool';
37+
import { getElementsTool, getElementsToolDefinition } from './tools/get-elements.tool';
3738
import { launchChromeTool, launchChromeToolDefinition } from './tools/launch-chrome.tool';
3839
import { emulateDeviceTool, emulateDeviceToolDefinition } from './tools/emulate-device.tool';
3940
import { withRecording } from './recording/step-recorder';
4041
import {
41-
sessionsIndexResource,
42-
sessionCurrentStepsResource,
42+
accessibilityResource,
43+
appStateResource,
44+
contextResource,
45+
contextsResource,
46+
cookiesResource,
47+
elementsResource,
48+
geolocationResource,
49+
screenshotResource,
50+
sessionCodeResource,
4351
sessionCurrentCodeResource,
52+
sessionCurrentStepsResource,
53+
sessionsIndexResource,
4454
sessionStepsResource,
45-
sessionCodeResource,
46-
} from './resources/sessions.resource';
47-
import { elementsResource } from './resources/elements.resource';
48-
import { accessibilityResource } from './resources/accessibility.resource';
49-
import { screenshotResource } from './resources/screenshot.resource';
50-
import { cookiesResource } from './resources/cookies.resource';
51-
import { appStateResource } from './resources/app-state.resource';
52-
import { contextsResource, contextResource } from './resources/contexts.resource';
53-
import { geolocationResource } from './resources/geolocation.resource';
54-
import { tabsResource } from './resources/tabs.resource';
55-
import { startSessionTool, startSessionToolDefinition, closeSessionTool, closeSessionToolDefinition } from './tools/session.tool';
55+
tabsResource,
56+
} from './resources';
57+
import {
58+
closeSessionTool,
59+
closeSessionToolDefinition,
60+
startSessionTool,
61+
startSessionToolDefinition
62+
} from './tools/session.tool';
5663
import { switchTabTool, switchTabToolDefinition } from './tools/tabs.tool';
5764

5865
console.log = (...args) => console.error('[LOG]', ...args);
@@ -125,6 +132,7 @@ registerTool(hideKeyboardToolDefinition, hideKeyboardTool);
125132
registerTool(setGeolocationToolDefinition, setGeolocationTool);
126133

127134
registerTool(executeScriptToolDefinition, executeScriptTool);
135+
registerTool(getElementsToolDefinition, getElementsTool);
128136

129137
registerTool(executeSequenceToolDefinition, withRecording('execute_sequence', executeSequenceTool));
130138

‎src/tools/get-elements.tool.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import { z } from 'zod';
2+
import type { ToolCallback } from '@modelcontextprotocol/sdk/server/mcp';
3+
import type { ToolDefinition } from '../types/tool';
4+
import { getBrowser } from '../session/state';
5+
import { getElements } from '../scripts/get-elements';
6+
import { encode } from '@toon-format/toon';
7+
import { coerceBoolean } from '../utils/zod-helpers';
8+
9+
export const getElementsToolDefinition: ToolDefinition = {
10+
name: 'get_elements',
11+
description: 'Get interactable elements on the current page. Use when wdio://session/current/elements does not return the desired elements.',
12+
inputSchema: {
13+
inViewportOnly: coerceBoolean.optional().default(false).describe('Only return elements visible in the current viewport (default: false).'),
14+
includeContainers: coerceBoolean.optional().default(false).describe('Include container elements like divs and sections (default: false)'),
15+
includeBounds: coerceBoolean.optional().default(false).describe('Include element bounding box coordinates (default: false)'),
16+
limit: z.number().optional().default(0).describe('Maximum number of elements to return (0 = no limit)'),
17+
offset: z.number().optional().default(0).describe('Number of elements to skip (for pagination)'),
18+
},
19+
};
20+
21+
export const getElementsTool: ToolCallback = async ({
22+
inViewportOnly = false,
23+
includeContainers = false,
24+
includeBounds = false,
25+
limit = 0,
26+
offset = 0,
27+
}: {
28+
inViewportOnly?: boolean;
29+
includeContainers?: boolean;
30+
includeBounds?: boolean;
31+
limit?: number;
32+
offset?: number;
33+
}) => {
34+
try {
35+
const browser = getBrowser();
36+
const result = await getElements(browser, { inViewportOnly, includeContainers, includeBounds, limit, offset });
37+
const text = encode(result).replace(/,""/g, ',').replace(/"",/g, ',');
38+
return { content: [{ type: 'text' as const, text }] };
39+
} catch (e) {
40+
return { isError: true as const, content: [{ type: 'text' as const, text: `Error getting elements: ${e}` }] };
41+
}
42+
};

‎tests/scripts/get-visible-elements.test.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { beforeEach, describe, expect, it, vi } from 'vitest';
2+
import { getInteractableBrowserElements } from '../../src/scripts/get-interactable-browser-elements';
3+
import { getMobileVisibleElements } from '../../src/scripts/get-visible-mobile-elements';
4+
import { getElements } from '../../src/scripts/get-elements';
5+
6+
vi.mock('../../src/scripts/get-interactable-browser-elements', () => ({
7+
getInteractableBrowserElements: vi.fn(),
8+
}));
9+
10+
vi.mock('../../src/scripts/get-visible-mobile-elements', () => ({
11+
getMobileVisibleElements: vi.fn(),
12+
}));
13+
14+
const mockGetElements = getInteractableBrowserElements as ReturnType<typeof vi.fn>;
15+
const mockGetMobile = getMobileVisibleElements as ReturnType<typeof vi.fn>;
16+
17+
function makeEl(name: string, inViewport = true) {
18+
return { name, selector: `#${name}`, tag: 'button', isInViewport: inViewport };
19+
}
20+
21+
const browserMock = { isAndroid: false, isIOS: false } as unknown as WebdriverIO.Browser;
22+
const androidMock = { isAndroid: true, isIOS: false } as unknown as WebdriverIO.Browser;
23+
24+
beforeEach(() => vi.clearAllMocks());
25+
26+
describe('getElements', () => {
27+
it('filters to viewport-only elements by default', async () => {
28+
mockGetElements.mockResolvedValue([makeEl('a', true), makeEl('b', false)]);
29+
const result = await getElements(browserMock, {});
30+
expect(result.total).toBe(1);
31+
expect(result.elements).toHaveLength(1);
32+
});
33+
34+
it('returns all elements when inViewportOnly is false', async () => {
35+
mockGetElements.mockResolvedValue([makeEl('a', true), makeEl('b', false)]);
36+
const result = await getElements(browserMock, { inViewportOnly: false });
37+
expect(result.total).toBe(2);
38+
});
39+
40+
it('applies limit and offset', async () => {
41+
mockGetElements.mockResolvedValue([makeEl('a'), makeEl('b'), makeEl('c')]);
42+
const result = await getElements(browserMock, { limit: 2, offset: 1 });
43+
expect(result.showing).toBe(2);
44+
expect(result.hasMore).toBe(false);
45+
expect(result.elements[0]).toMatchObject({ name: 'b' });
46+
});
47+
48+
it('reports hasMore correctly when more elements remain', async () => {
49+
mockGetElements.mockResolvedValue([makeEl('a'), makeEl('b'), makeEl('c')]);
50+
const result = await getElements(browserMock, { limit: 1, offset: 0 });
51+
expect(result.hasMore).toBe(true);
52+
});
53+
54+
it('delegates to getMobileVisibleElements on Android', async () => {
55+
mockGetMobile.mockResolvedValue([makeEl('btn')]);
56+
await getElements(androidMock, {});
57+
expect(mockGetMobile).toHaveBeenCalledWith(androidMock, 'android', expect.any(Object));
58+
expect(mockGetElements).not.toHaveBeenCalled();
59+
});
60+
});

‎tests/tools/get-elements-tool.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import { beforeEach, describe, expect, it, vi } from 'vitest';
2+
import { getElements } from '../../src/scripts/get-elements';
3+
import { getBrowser } from '../../src/session/state';
4+
import { getElementsTool } from '../../src/tools/get-elements.tool';
5+
6+
vi.mock('../../src/scripts/get-elements', () => ({
7+
getElements: vi.fn(),
8+
}));
9+
10+
vi.mock('../../src/session/state', () => ({
11+
getBrowser: vi.fn(),
12+
getState: vi.fn(() => ({
13+
browsers: new Map(),
14+
currentSession: null,
15+
sessionMetadata: new Map(),
16+
sessionHistory: new Map(),
17+
})),
18+
}));
19+
20+
type ToolFn = (args: Record<string, unknown>) => Promise<{
21+
content: { type: string; text: string }[];
22+
isError?: boolean
23+
}>;
24+
const callTool = getElementsTool as unknown as ToolFn;
25+
26+
const mockGetVisible = getElements as ReturnType<typeof vi.fn>;
27+
const mockGetBrowser = getBrowser as ReturnType<typeof vi.fn>;
28+
29+
const defaultResult = { total: 1, showing: 1, hasMore: false, elements: [{ name: 'btn', selector: '#btn' }] };
30+
31+
beforeEach(() => {
32+
vi.clearAllMocks();
33+
mockGetBrowser.mockReturnValue({ isAndroid: false, isIOS: false });
34+
mockGetVisible.mockResolvedValue(defaultResult);
35+
});
36+
37+
describe('get_elements tool', () => {
38+
it('passes inViewportOnly false to getElements', async () => {
39+
await callTool({ inViewportOnly: false });
40+
expect(mockGetVisible).toHaveBeenCalledWith(
41+
expect.anything(),
42+
expect.objectContaining({ inViewportOnly: false })
43+
);
44+
});
45+
46+
it('returns toon-encoded text with element data', async () => {
47+
const result = await callTool({});
48+
expect(result.isError).toBeFalsy();
49+
expect(result.content[0].text).toContain('btn');
50+
});
51+
52+
it('returns isError true on failure', async () => {
53+
mockGetVisible.mockRejectedValue(new Error('browser disconnected'));
54+
const result = await callTool({});
55+
expect(result.isError).toBe(true);
56+
expect(result.content[0].text).toContain('browser disconnected');
57+
});
58+
59+
it('passes limit and offset to getElements', async () => {
60+
await callTool({ limit: 10, offset: 5 });
61+
expect(mockGetVisible).toHaveBeenCalledWith(
62+
expect.anything(),
63+
expect.objectContaining({ limit: 10, offset: 5 })
64+
);
65+
});
66+
});

0 commit comments

Comments
 (0)