Add auto-advance: click Next, scroll to top, capture next page

After each capture the controller asks the vision model to locate a 'Next'
control (returned as normalized screen coordinates), clicks it via OS input,
scrolls to top, and continues the loop. Configurable via SCREEN_LEADS_AUTO_NEXT
/ _NEXT_LOAD_PAUSE / _MAX_AUTO_NEXT, with a per-run safety cap.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-27 07:56:13 +05:30
parent c1eef4c6aa
commit 511e36b660
6 changed files with 93 additions and 0 deletions
+36
View File
@@ -88,3 +88,39 @@ def classify_page(screenshot_path: str) -> dict[str, Any]:
output_config={"format": {"type": "json_schema", "schema": _CLASSIFY_SCHEMA}},
)
return _first_json(response)
_NEXT_SCHEMA = {
"type": "object",
"properties": {
"found": {"type": "boolean"},
"x": {"type": "number"}, # 0..1 fraction of image width (button centre)
"y": {"type": "number"}, # 0..1 fraction of image height
"label": {"type": "string"},
},
"required": ["found", "x", "y", "label"],
"additionalProperties": False,
}
_NEXT_PROMPT = (
"Look at this screenshot for a control that advances to the NEXT item or page — "
"for example a button or link labelled 'Next', 'Next result', 'See next profile', "
"or a right-facing pagination arrow ('>' / '' / ''). Ignore 'Back'/'Previous' "
"and unrelated arrows. If such a control is clearly visible, set found=true and give "
"its CENTRE position as fractions of the image: x = left→right (0.01.0), "
"y = top→bottom (0.01.0), plus its visible label. If none is visible, "
"return found=false, x=0, y=0, label=\"\"."
)
def find_next_button(screenshot_path: str) -> dict[str, Any]:
response = _client().messages.create(
model=ANTHROPIC_MODEL,
max_tokens=256,
messages=[{
"role": "user",
"content": [_image_block(screenshot_path), {"type": "text", "text": _NEXT_PROMPT}],
}],
output_config={"format": {"type": "json_schema", "schema": _NEXT_SCHEMA}},
)
return _first_json(response)