diff --git a/README.md b/README.md index a94ff89..c4ae52d 100644 --- a/README.md +++ b/README.md @@ -60,5 +60,6 @@ Check out the [examples](./examples) directory for complete working examples: | [scheduling.ts](./examples/scheduling.ts) | Set up robot schedules to execute runs | | [webhooks.ts](./examples/webhooks.ts) | Configure webhook notifications | | [robot-management.ts](./examples/robot-management.ts) | CRUD operations for robots | +| [list-limit.ts](./examples/list-limit.ts) | Change a robot's limit without resending its workflow | | [complete-workflow.ts](./examples/complete-workflow.ts) | Create a robot combining multiple features | diff --git a/examples/list-limit.ts b/examples/list-limit.ts new file mode 100644 index 0000000..9d8f897 --- /dev/null +++ b/examples/list-limit.ts @@ -0,0 +1,113 @@ +/** + * List Limit Example + * + * This example demonstrates: + * - Changing a robot's limit without resending its workflow + * - Doing so for extract, crawl, and search robots + * - Targeting a specific list on a robot that has several + * + * Only the limit is sent to the backend. Selectors, pagination, crawl depth, + * search filters, and everything else are left exactly as they were. + */ + +import 'dotenv/config'; +import { Extract, Crawl, Search, Client, Robot } from 'maxun-sdk'; + +const config = { + apiKey: process.env.MAXUN_API_KEY!, + baseUrl: process.env.MAXUN_BASE_URL, +}; + +/** The stored config object holding the limit, for a given action. */ +function configOf(robot: Robot, action: string) { + return (robot.getData().recording?.workflow || []) + .flatMap((pair: any) => pair.what || []) + .filter((a: any) => a.action === action) + .flatMap((a: any) => a.args || []) + .find((arg: any) => arg && typeof arg === 'object' && 'limit' in arg); +} + +/** Prints the limit alongside the settings that sit next to it. */ +function describe(label: string, robot: Robot, action: string, neighbours: string[]) { + const cfg = configOf(robot, action) || {}; + const rest = neighbours.map((k) => `${k}=${JSON.stringify(cfg[k])}`).join(', '); + console.log(` ${label.padEnd(7)} limit=${String(cfg.limit).padEnd(4)} ${rest}`); +} + +async function main() { + const extractor = new Extract(config); + + try { + // --- extract robot: how many items the list collects ----------------- + console.log('\nExtract robot (scrapeList)'); + + const robot = await extractor + .create(`Books Scraper ${Date.now()}`) + .navigate('https://books.toscrape.com/') + .captureList({ selector: 'article.product_pod', maxItems: 10 }); + + describe('before', robot, 'scrapeList', ['listSelector']); + await robot.setListLimit(25); + describe('after', robot, 'scrapeList', ['listSelector']); + + // --- crawl robot: how many pages it visits --------------------------- + console.log('\nCrawl robot (crawl)'); + + const crawler = await new Crawl(config).create( + `Site Crawler ${Date.now()}`, + 'https://books.toscrape.com/', + { mode: 'domain', limit: 15, maxDepth: 2 } + ); + + describe('before', crawler, 'crawl', ['mode', 'maxDepth']); + await crawler.setListLimit(50); + describe('after', crawler, 'crawl', ['mode', 'maxDepth']); + + // --- search robot: how many results it returns ----------------------- + console.log('\nSearch robot (search)'); + + const searcher = await new Search(config).create(`Web Search ${Date.now()}`, { + query: 'web scraping', + mode: 'discover', + limit: 8, + }); + + describe('before', searcher, 'search', ['query', 'provider']); + await searcher.setListLimit(20); + describe('after', searcher, 'search', ['query', 'provider']); + + /** + * setListLimit updates the first action it finds that carries a limit. + * For a robot with more than one list, use the client directly and name + * the position. Positions are assigned server-side, so read them from the + * robot rather than assuming them. + */ + console.log('\nUpdating by explicit position'); + + const client = new Client(config); + const workflow = robot.getData().recording?.workflow || []; + + workflow.forEach((pair: any, pairIndex: number) => { + (pair.what || []).forEach((action: any, actionIndex: number) => { + (action.args || []).forEach((arg: any, argIndex: number) => { + if (arg && typeof arg === 'object' && 'limit' in arg) { + console.log( + ` found ${action.action} limit=${arg.limit} at pair ${pairIndex}, action ${actionIndex}, arg ${argIndex}` + ); + } + }); + }); + }); + + await client.updateListLimits(robot.id, [ + { pairIndex: 0, actionIndex: 0, argIndex: 0, limit: 50 }, + ]); + + const updated = await extractor.getRobot(robot.id); + describe('after', updated, 'scrapeList', ['listSelector']); + } catch (error: any) { + console.error('Failed:', error.message); + } +} + +main(); diff --git a/src/client/maxun-client.ts b/src/client/maxun-client.ts index 706f889..e4f3a32 100644 --- a/src/client/maxun-client.ts +++ b/src/client/maxun-client.ts @@ -22,6 +22,8 @@ import { CrawlOptions, SearchOptions, LlmOptions, + ListLimitUpdate, //added + } from '../types'; /** @@ -165,6 +167,22 @@ export class Client { return response.data.data; } + + /** + * Update one or more list limits without resending the whole workflow. + */ + async updateListLimits(robotId: string, limits: ListLimitUpdate[]): Promise { + const response = await this.axios.put>( + `/robots/${robotId}`, + { limits } + ); + if (!response.data.data) { + throw new MaxunError(`Failed to update list limits for robot ${robotId}`); + } + return response.data.data; + } + + /** * Delete a robot */ diff --git a/src/robot/robot.ts b/src/robot/robot.ts index dbe1d10..5ca9359 100644 --- a/src/robot/robot.ts +++ b/src/robot/robot.ts @@ -2,7 +2,7 @@ * Robot class - represents a saved workflow that can be executed */ -import { RunResult, RobotData, ScheduleConfig, WebhookConfig, ExecutionOptions, Run } from '../types'; +import { RunResult, RobotData, ScheduleConfig, WebhookConfig, ExecutionOptions, Run, MaxunError } from '../types'; import { Client } from '../client/maxun-client'; export class Robot { @@ -110,6 +110,42 @@ export class Robot { this.robotData = updated; } + /** + * Set the maximum number of items this robot collects. + * + * Applies to the three actions that carry a limit: `scrapeList` on extract + * robots, `crawl` on crawl robots, and `search` on search robots. The action + * is located automatically, so callers do not need to know its position in + * the workflow. Only the limit is sent; the rest of the workflow is untouched. + * + * @throws MaxunError if the robot has no action with a limit. + */ + async setListLimit(limit: number): Promise { + const LIMIT_ACTIONS = ['scrapeList', 'crawl', 'search']; + const workflow = this.robotData.recording?.workflow || []; + + for (let p = 0; p < workflow.length; p++) { + const what = workflow[p].what || []; + for (let a = 0; a < what.length; a++) { + if (!LIMIT_ACTIONS.includes(what[a].action)) continue; + const args = what[a].args || []; + for (let g = 0; g < args.length; g++) { + const arg = args[g]; + if (arg && typeof arg === 'object' && 'limit' in arg) { + this.robotData = await this.client.updateListLimits(this.id, [ + { pairIndex: p, actionIndex: a, argIndex: g, limit }, + ]); + return; + } + } + } + } + + throw new MaxunError('This robot has no scrapeList, crawl, or search action with a limit to update.'); + } + + + /** * Get all webhooks for this robot */ diff --git a/src/types/index.ts b/src/types/index.ts index a98013e..f07ac7b 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -78,6 +78,20 @@ export interface RobotData { updatedAt?: string; } + +/** + * Coordinates of a single list limit within a robot's workflow, + * plus the new value to set. + */ +export interface ListLimitUpdate { + pairIndex: number; + actionIndex: number; + argIndex: number; + limit: number; +} + + + export interface Run { id: string; status: RunStatus;