|
| 1 | +"""Propose a clean element list from raw pixels, with no template or model. |
| 2 | +
|
| 3 | +Set-of-Marks, ``observation`` and the grounding helpers all assume you already |
| 4 | +have a list of element boxes — but on a screen the framework doesn't model |
| 5 | +(a game, a custom-drawn app, a remote desktop) there is no accessibility tree to |
| 6 | +provide one. ``element_proposal`` builds that top-of-funnel list from pixels: |
| 7 | +detect candidate *widget* boxes (closed-edge blobs) and *text* boxes |
| 8 | +(:func:`text_regions.find_text_regions`), fuse them — dropping widget boxes that |
| 9 | +are really just text — and return them in reading order, each tagged ``text`` or |
| 10 | +``widget``. |
| 11 | +
|
| 12 | +* :func:`propose_elements` — the full pixel-to-elements pipeline. |
| 13 | +* :func:`tag_kinds` — pure: label fused boxes ``text`` / ``widget`` by source and |
| 14 | + keep their reading-order ``index``. |
| 15 | +
|
| 16 | +The fusion / cross-check / ordering reuse :mod:`element_parse` (the ``ocr`` > |
| 17 | +``icon`` priority *is* the "drop widget-that-is-really-text" check) and |
| 18 | +:mod:`text_regions`; ``cv2`` is imported lazily so the module stays importable. |
| 19 | +:func:`tag_kinds` is pure and fully testable. Imports no ``PySide6``. |
| 20 | +""" |
| 21 | +from typing import Any, Dict, List, Optional, Sequence |
| 22 | + |
| 23 | +# Reading-order source tag to element kind. |
| 24 | +_KIND_BY_SOURCE = {"ocr": "text", "icon": "widget", "a11y": "element"} |
| 25 | + |
| 26 | + |
| 27 | +def tag_kinds(elements: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]: |
| 28 | + """Label fused boxes ``text`` / ``widget`` by source (pure). |
| 29 | +
|
| 30 | + Each input box carries a ``source`` (``ocr`` / ``icon``) and an ``index`` |
| 31 | + from :func:`element_parse.reading_order`. Returns ``[{box, kind, index}]``. |
| 32 | + """ |
| 33 | + result: List[Dict[str, Any]] = [] |
| 34 | + for element in elements: |
| 35 | + box = [int(element["x"]), int(element["y"]), |
| 36 | + int(element["width"]), int(element["height"])] |
| 37 | + kind = _KIND_BY_SOURCE.get(element.get("source"), "widget") |
| 38 | + result.append({"box": box, "kind": kind, "index": element.get("index")}) |
| 39 | + return result |
| 40 | + |
| 41 | + |
| 42 | +def _reasonable(box: Dict[str, Any], frame_w: int, frame_h: int) -> bool: |
| 43 | + """Keep plausibly-widget blobs: not the whole frame, not a thin rule.""" |
| 44 | + width, height = int(box["width"]), int(box["height"]) |
| 45 | + if width >= 0.95 * frame_w and height >= 0.95 * frame_h: |
| 46 | + return False |
| 47 | + aspect = width / height if height else 0.0 |
| 48 | + return 0.05 <= aspect <= 15.0 |
| 49 | + |
| 50 | + |
| 51 | +def _widget_boxes(gray: Any, min_area: int) -> List[Dict[str, Any]]: |
| 52 | + """Detect candidate widget boxes as closed-edge blobs (cv2).""" |
| 53 | + import cv2 |
| 54 | + from je_auto_control.utils.cv2_utils.blobs import connected_boxes |
| 55 | + edges = cv2.Canny(gray, 50, 150) |
| 56 | + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5)) |
| 57 | + closed = cv2.morphologyEx(edges, cv2.MORPH_CLOSE, kernel) |
| 58 | + height, width = gray.shape[:2] |
| 59 | + return [box for box in connected_boxes(closed, min_area=int(min_area)) |
| 60 | + if _reasonable(box, width, height)] |
| 61 | + |
| 62 | + |
| 63 | +def propose_elements(source: Optional[Any] = None, *, |
| 64 | + region: Optional[Sequence[int]] = None, min_area: int = 80, |
| 65 | + iou_threshold: float = 0.5) -> List[Dict[str, Any]]: |
| 66 | + """Propose ``text`` / ``widget`` element boxes from pixels, in reading order. |
| 67 | +
|
| 68 | + Detects widget blobs and text regions on ``source`` (a fresh screen grab of |
| 69 | + ``region`` by default), fuses them (overlapping text wins over widget), and |
| 70 | + orders them. Returns ``[{box, kind, index}]``. |
| 71 | + """ |
| 72 | + from je_auto_control.utils.element_parse import fuse_elements, reading_order |
| 73 | + from je_auto_control.utils.text_regions import find_text_regions |
| 74 | + from je_auto_control.utils.visual_match.visual_match import _haystack_gray |
| 75 | + gray = _haystack_gray(source, region) |
| 76 | + text = find_text_regions(gray, min_area=int(min_area)) |
| 77 | + widgets = _widget_boxes(gray, int(min_area)) |
| 78 | + fused = fuse_elements(ocr_boxes=text, icon_boxes=widgets, |
| 79 | + iou_threshold=float(iou_threshold)) |
| 80 | + return tag_kinds(reading_order(fused)) |
0 commit comments