|
| 1 | +"""Classify what kind of widget a box is from its pixel shape. |
| 2 | +
|
| 3 | +Set-of-Marks and element proposers hand back *boxes*, but not *what each box is*. |
| 4 | +``form_fields.checkbox_state`` already reads a box known to be a checkbox; the |
| 5 | +gap is the typing step before it — is this box a checkbox, a radio button, a |
| 6 | +push button, a text field or a toggle? ``icon_classify`` answers that from cheap |
| 7 | +geometric features (no model): |
| 8 | +
|
| 9 | +* :func:`box_features` — extract ``{aspect, fill, edge_density, circularity}`` |
| 10 | + for a box region (the objective measurements). |
| 11 | +* :func:`classify_widget` — pure: map a feature dict to a widget type by |
| 12 | + documented heuristics. |
| 13 | +* :func:`classify_icon` — compose the two: a box to ``{type, features}``. |
| 14 | +
|
| 15 | +``classify_widget`` is pure and fully testable; ``box_features`` imports cv2 / |
| 16 | +numpy lazily (the module stays importable without them) and reuses |
| 17 | +:func:`visual_match._to_gray`. Imports no ``PySide6``. |
| 18 | +""" |
| 19 | +from typing import Any, Dict, Sequence |
| 20 | + |
| 21 | +# The widget types this classifier can return. |
| 22 | +WIDGET_TYPES = ("radio", "toggle", "checkbox", "text_field", "button", "icon") |
| 23 | + |
| 24 | + |
| 25 | +def _is_round(aspect: float, circ: float) -> bool: |
| 26 | + """Near-square and circular (a radio button / round dot).""" |
| 27 | + return 0.7 <= aspect <= 1.4 and circ >= 0.7 |
| 28 | + |
| 29 | + |
| 30 | +def _is_pill(aspect: float, circ: float) -> bool: |
| 31 | + """Wide and rounded (a toggle switch).""" |
| 32 | + return 1.8 <= aspect <= 3.5 and circ >= 0.55 |
| 33 | + |
| 34 | + |
| 35 | +def classify_widget(features: Dict[str, float]) -> str: |
| 36 | + """Map geometric ``features`` to a widget type by heuristics (pure). |
| 37 | +
|
| 38 | + Uses ``aspect`` (w/h), ``circularity`` (1 = circle), and ``fill`` (ink |
| 39 | + fraction). Round → ``radio``; wide & rounded → ``toggle``; near-square & |
| 40 | + sparse → ``checkbox``; wide & hollow → ``text_field``; wide & filled → |
| 41 | + ``button``; otherwise ``icon``. |
| 42 | + """ |
| 43 | + aspect = float(features.get("aspect", 1.0)) |
| 44 | + circ = float(features.get("circularity", 0.0)) |
| 45 | + fill = float(features.get("fill", 0.0)) |
| 46 | + if _is_round(aspect, circ): |
| 47 | + return "radio" |
| 48 | + if _is_pill(aspect, circ): |
| 49 | + return "toggle" |
| 50 | + if 0.7 <= aspect <= 1.4 and fill <= 0.6: |
| 51 | + return "checkbox" |
| 52 | + if aspect >= 2.5 and fill <= 0.2: |
| 53 | + return "text_field" |
| 54 | + if aspect >= 1.5 and fill >= 0.2: |
| 55 | + return "button" |
| 56 | + return "icon" |
| 57 | + |
| 58 | + |
| 59 | +def _circularity(binary: Any) -> float: |
| 60 | + """Circularity (``4*pi*A / P^2``, 1 = circle) of the largest blob.""" |
| 61 | + import math |
| 62 | + |
| 63 | + import cv2 |
| 64 | + contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, |
| 65 | + cv2.CHAIN_APPROX_SIMPLE) |
| 66 | + if not contours: |
| 67 | + return 0.0 |
| 68 | + largest = max(contours, key=cv2.contourArea) |
| 69 | + area = float(cv2.contourArea(largest)) |
| 70 | + perimeter = float(cv2.arcLength(largest, True)) |
| 71 | + if perimeter <= 0.0: |
| 72 | + return 0.0 |
| 73 | + return min(1.0, 4.0 * math.pi * area / (perimeter * perimeter)) |
| 74 | + |
| 75 | + |
| 76 | +def box_features(source: Any, box: Sequence[int]) -> Dict[str, float]: |
| 77 | + """Extract ``{aspect, fill, edge_density, circularity}`` for a box (cv2). |
| 78 | +
|
| 79 | + ``aspect`` is width/height, ``fill`` the ink fraction (Otsu foreground), |
| 80 | + ``edge_density`` the Canny-edge fraction, ``circularity`` the largest blob's |
| 81 | + roundness. An empty box yields all zeros. |
| 82 | + """ |
| 83 | + import cv2 |
| 84 | + from je_auto_control.utils.visual_match.visual_match import _to_gray |
| 85 | + gray = _to_gray(source) |
| 86 | + x, y, w, h = (int(box[0]), int(box[1]), int(box[2]), int(box[3])) |
| 87 | + patch = gray[max(0, y):y + h, max(0, x):x + w] |
| 88 | + if patch.size == 0: |
| 89 | + return {"aspect": 0.0, "fill": 0.0, "edge_density": 0.0, |
| 90 | + "circularity": 0.0} |
| 91 | + _, binary = cv2.threshold(patch, 0, 255, |
| 92 | + cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) |
| 93 | + fill = float((binary > 0).sum()) / patch.size |
| 94 | + edges = cv2.Canny(patch, 50, 150) |
| 95 | + edge_density = float((edges > 0).sum()) / patch.size |
| 96 | + return { |
| 97 | + "aspect": round(w / h, 3) if h else 0.0, |
| 98 | + "fill": round(fill, 3), |
| 99 | + "edge_density": round(edge_density, 3), |
| 100 | + "circularity": round(_circularity(binary), 3), |
| 101 | + } |
| 102 | + |
| 103 | + |
| 104 | +def classify_icon(source: Any, box: Sequence[int]) -> Dict[str, Any]: |
| 105 | + """Classify the widget in a box from its pixels: ``{type, features}``.""" |
| 106 | + features = box_features(source, box) |
| 107 | + return {"type": classify_widget(features), "features": features} |
0 commit comments