From 5c4a43c5e1e243bf34aa2480beb147a9b85587b4 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 04:16:46 +0800 Subject: [PATCH] Add edge_match: edge-shape (Chamfer) template matching Intensity NCC drops when a control is re-filled or re-themed, and ORB needs corner texture flat-design glyphs lack. Match by edge shape instead: Canny both images, distance-transform the scene edges, slide the template's edges over it and score by mean edge-to-edge distance (Chamfer). A perfect outline aligns at ~0 cost regardless of fill. Reuses visual_match's loaders/resize/NMS/Match and edge_lines's Canny default. --- README/WHATS_NEW_zh-CN.md | 6 + README/WHATS_NEW_zh-TW.md | 6 + WHATS_NEW.md | 6 + .../doc/new_features/v168_features_doc.rst | 47 ++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v168_features_doc.rst | 42 +++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 7 ++ .../gui/script_builder/command_schema.py | 25 +++++ je_auto_control/utils/edge_match/__init__.py | 6 + .../utils/edge_match/edge_match.py | 106 ++++++++++++++++++ .../utils/executor/action_executor.py | 27 +++++ .../utils/mcp_server/tools/_factories.py | 31 +++++ .../utils/mcp_server/tools/_handlers.py | 11 ++ .../headless/test_edge_match_batch.py | 66 +++++++++++ 15 files changed, 388 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v168_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v168_features_doc.rst create mode 100644 je_auto_control/utils/edge_match/__init__.py create mode 100644 je_auto_control/utils/edge_match/edge_match.py create mode 100644 test/unit_test/headless/test_edge_match_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 661538b3..0a082e6b 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 边缘形状(Chamfer)模板匹配 + +以轮廓定位扁平图标,对填充 / 主题 / 抗锯齿稳健。完整参考:[`docs/source/Zh/doc/new_features/v168_features_doc.rst`](../docs/source/Zh/doc/new_features/v168_features_doc.rst)。 + +- **`edge_match` / `edge_match_all` / `chamfer_distance`**(`AC_edge_match`、`AC_edge_match_all`):强度 NCC(`visual_match`)在控件换填充 / 换主题时分数下降,ORB(`feature_match`)需要扁平图标缺乏的角点纹理。本功能以*边缘形状*匹配:对两图跑 Canny,对场景边缘做距离变换,把模板边缘滑过它并以平均边缘间距(Chamfer)评分。完美轮廓不论填充皆以约 0 成本对齐。重用 `visual_match` 的加载器 / resize / NMS / `Match` 与 `edge_lines` 的 Canny 默认。`haystack` 可注入;不导入 `PySide6`。 + ## 本次更新 (2026-06-24) — 动作效果分类(我的点击有没有效果?) 告诉代理点击有没有效果——以及是否发生在它瞄准之处。完整参考:[`docs/source/Zh/doc/new_features/v167_features_doc.rst`](../docs/source/Zh/doc/new_features/v167_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index e88d9b23..c4c17a1e 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 邊緣形狀(Chamfer)樣板比對 + +以輪廓定位扁平圖示,對填充 / 主題 / 抗鋸齒穩健。完整參考:[`docs/source/Zh/doc/new_features/v168_features_doc.rst`](../docs/source/Zh/doc/new_features/v168_features_doc.rst)。 + +- **`edge_match` / `edge_match_all` / `chamfer_distance`**(`AC_edge_match`、`AC_edge_match_all`):強度 NCC(`visual_match`)在控制項換填充 / 換主題時分數下降,ORB(`feature_match`)需要扁平圖示缺乏的角點紋理。本功能以*邊緣形狀*比對:對兩圖跑 Canny,對場景邊緣做距離轉換,把樣板邊緣滑過它並以平均邊緣間距(Chamfer)評分。完美輪廓不論填充皆以約 0 成本對齊。重用 `visual_match` 的載入器 / resize / NMS / `Match` 與 `edge_lines` 的 Canny 預設。`haystack` 可注入;不匯入 `PySide6`。 + ## 本次更新 (2026-06-24) — 動作效果分類(我的點擊有沒有效果?) 告訴代理點擊有沒有效果——以及是否發生在它瞄準之處。完整參考:[`docs/source/Zh/doc/new_features/v167_features_doc.rst`](../docs/source/Zh/doc/new_features/v167_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 862dac75..1ab6e0c2 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Edge-Shape (Chamfer) Template Matching + +Locate flat icons by outline, robust to fill / theme / anti-aliasing. Full reference: [`docs/source/Eng/doc/new_features/v168_features_doc.rst`](docs/source/Eng/doc/new_features/v168_features_doc.rst). + +- **`edge_match` / `edge_match_all` / `chamfer_distance`** (`AC_edge_match`, `AC_edge_match_all`): intensity NCC (`visual_match`) drops when a control is re-filled / re-themed, and ORB (`feature_match`) needs corner texture flat-design glyphs lack. This matches by *edge shape*: Canny both images, distance-transform the scene edges, slide the template's edges over it and score by mean edge-to-edge distance (Chamfer). A perfect outline aligns at ~0 cost regardless of fill. Reuses `visual_match`'s loaders / resize / NMS / `Match` and `edge_lines`'s Canny default. Injectable `haystack`; no `PySide6`. + ## What's new (2026-06-24) — Action-Effect Classification (Did My Click Do Anything?) Tell an agent whether a click did anything — and whether it happened where it aimed. Full reference: [`docs/source/Eng/doc/new_features/v167_features_doc.rst`](docs/source/Eng/doc/new_features/v167_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v168_features_doc.rst b/docs/source/Eng/doc/new_features/v168_features_doc.rst new file mode 100644 index 00000000..25be2ca8 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v168_features_doc.rst @@ -0,0 +1,47 @@ +Edge-Shape (Chamfer) Template Matching +====================================== + +Intensity correlation (``visual_match``) is dragged down when the same control is rendered +with a different fill, gradient, theme or anti-aliasing, and ORB feature matching +(``feature_match``) needs corner texture that flat-design glyphs — a hamburger menu, a plain +chevron — simply do not have. ``edge_match`` locates a template by its *edge shape* instead: +it runs Canny on both images, builds a distance transform of the scene edges, and slides the +template's edges over it, scoring each position by the mean distance from a template edge to +the nearest scene edge (Chamfer matching). A perfect alignment costs ~0 regardless of how the +shape is filled or shaded. + +It reuses ``visual_match``'s gray loaders / resize / NMS / ``Match`` and ``edge_lines``'s Canny +default, so no matching or geometry code is duplicated. The ``haystack`` is injectable +(ndarray / path / PIL); the search is unit-testable on synthetic arrays. Imports no +``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import edge_match, edge_match_all, chamfer_distance + + # find a flat icon regardless of its fill colour / theme + hit = edge_match("chevron.png", min_score=0.7) + if hit: + click(*hit.center) + + for m in edge_match_all("divider_handle.png", min_score=0.8): + print(m.center, m.score) + + print(chamfer_distance("logo_outline.png")) # 0 = edges coincide + +``edge_match`` returns the best ``Match`` (score = ``1 / (1 + mean edge distance)``, so 1.0 is +a perfect outline match) over the requested ``scales``, or ``None``. ``edge_match_all`` returns +every match at or above ``min_score`` with overlaps removed by NMS. ``chamfer_distance`` returns +the mean edge-to-edge distance at the best alignment (0 = the outlines coincide). + +Executor commands +----------------- + +``AC_edge_match`` (``template`` / ``min_score`` / ``scales`` / ``region`` → +``{found, match}``) and ``AC_edge_match_all`` (adds ``max_results`` / ``nms_iou`` → +``{count, matches}``). They are exposed as the MCP tools ``ac_edge_match`` / +``ac_edge_match_all`` (read-only) and as the Script Builder commands **Match Template (edge +shape)** / **Match Template All (edge shape)** under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 0dfe9e20..2bec5e7c 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -190,6 +190,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v165_features_doc doc/new_features/v166_features_doc doc/new_features/v167_features_doc + doc/new_features/v168_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v168_features_doc.rst b/docs/source/Zh/doc/new_features/v168_features_doc.rst new file mode 100644 index 00000000..405a7df7 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v168_features_doc.rst @@ -0,0 +1,42 @@ +邊緣形狀(Chamfer)樣板比對 +============================ + +當同一個控制項以不同填充、漸層、主題或抗鋸齒繪製時,強度相關(``visual_match``)會被拉低; +而 ORB 特徵比對(``feature_match``)需要角點紋理,扁平設計的圖示——漢堡選單、單純的箭號 +——根本沒有。``edge_match`` 改以*邊緣形狀*定位樣板:對兩張影像跑 Canny,對場景邊緣建立 +距離轉換,再把樣板邊緣滑過它,以「每個樣板邊緣到最近場景邊緣的平均距離」為每個位置評分 +(Chamfer 比對)。完美對齊的成本約為 0,與形狀如何填充或著色無關。 + +本功能重用 ``visual_match`` 的灰階載入器 / resize / NMS / ``Match`` 與 ``edge_lines`` 的 +Canny 預設,因此不重複任何比對或幾何程式。``haystack`` 可注入(ndarray / 路徑 / PIL); +搜尋可在合成陣列上單元測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import edge_match, edge_match_all, chamfer_distance + + # 不論填色 / 主題,找出扁平圖示 + hit = edge_match("chevron.png", min_score=0.7) + if hit: + click(*hit.center) + + for m in edge_match_all("divider_handle.png", min_score=0.8): + print(m.center, m.score) + + print(chamfer_distance("logo_outline.png")) # 0 = 邊緣完全重合 + +``edge_match`` 在指定 ``scales`` 中回傳最佳 ``Match``(score = ``1 / (1 + 平均邊緣距離)``, +故 1.0 為完美輪廓匹配)或 ``None``。``edge_match_all`` 回傳所有達到 ``min_score`` 的匹配, +重疊以 NMS 移除。``chamfer_distance`` 回傳最佳對齊處的平均邊緣間距(0 = 輪廓重合)。 + +執行器指令 +---------- + +``AC_edge_match``(``template`` / ``min_score`` / ``scales`` / ``region`` → +``{found, match}``)與 ``AC_edge_match_all``(另加 ``max_results`` / ``nms_iou`` → +``{count, matches}``)。兩者以 MCP 工具 ``ac_edge_match`` / ``ac_edge_match_all``(唯讀)及 +Script Builder 指令 **Match Template (edge shape)** / **Match Template All (edge shape)** +(位於 **Image** 分類下)形式提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 84532add..5991fd02 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -190,6 +190,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v165_features_doc doc/new_features/v166_features_doc doc/new_features/v167_features_doc + doc/new_features/v168_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 27f5af59..bb1231bc 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -287,6 +287,10 @@ from je_auto_control.utils.match_trust import ( TrustedMatch, match_with_trust, score_peaks, ) +# Edge-shape (Chamfer / distance-transform) template matching +from je_auto_control.utils.edge_match import ( + chamfer_distance, edge_match, edge_match_all, +) # Otsu auto-thresholding for template matching (no hand-tuned min_score) from je_auto_control.utils.match_autothresh import ( auto_threshold, match_auto, @@ -1231,6 +1235,9 @@ def start_autocontrol_gui(*args, **kwargs): "score_peaks", "auto_threshold", "match_auto", + "edge_match", + "edge_match_all", + "chamfer_distance", "GridCell", "grid_cells", "cell_for_point", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 5f0cadbe..f9501265 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -371,6 +371,31 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: ), description="Derive a match threshold + separability from the score map.", )) + specs.append(CommandSpec( + "AC_edge_match", "Image", "Match Template (edge shape)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.7, + min_value=0.0, max_value=1.0), + FieldSpec("scales", FieldType.STRING, optional=True, + placeholder="[0.9, 1.0, 1.1]"), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Locate by edge shape (Chamfer) — robust to fill / theme / AA.", + )) + specs.append(CommandSpec( + "AC_edge_match_all", "Image", "Match Template All (edge shape)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.7, + min_value=0.0, max_value=1.0), + FieldSpec("max_results", FieldType.INT, optional=True, default=20), + FieldSpec("nms_iou", FieldType.FLOAT, optional=True, default=0.3, + min_value=0.0, max_value=1.0), + ), + description="Find every edge-shape (Chamfer) match (NMS-deduped).", + )) specs.append(CommandSpec( "AC_grid_cells", "Image", "Grid Cells (coarse grounding)", fields=( diff --git a/je_auto_control/utils/edge_match/__init__.py b/je_auto_control/utils/edge_match/__init__.py new file mode 100644 index 00000000..a15de81c --- /dev/null +++ b/je_auto_control/utils/edge_match/__init__.py @@ -0,0 +1,6 @@ +"""Edge-shape (Chamfer / distance-transform) template matching.""" +from je_auto_control.utils.edge_match.edge_match import ( + chamfer_distance, edge_match, edge_match_all, +) + +__all__ = ["edge_match", "edge_match_all", "chamfer_distance"] diff --git a/je_auto_control/utils/edge_match/edge_match.py b/je_auto_control/utils/edge_match/edge_match.py new file mode 100644 index 00000000..2cfacded --- /dev/null +++ b/je_auto_control/utils/edge_match/edge_match.py @@ -0,0 +1,106 @@ +"""Edge-shape (Chamfer / distance-transform) template matching. + +Intensity correlation (``visual_match``) is dragged down when the same control is rendered +with a different fill, gradient, theme or anti-aliasing, and ORB feature matching +(``feature_match``) needs corner texture that flat-design glyphs — a hamburger menu, a plain +chevron — simply do not have. ``edge_match`` locates a template by its *edge shape* instead: +it runs Canny on both images, builds a distance transform of the scene edges, and slides the +template's edges over it, scoring each position by the mean distance from a template edge to +the nearest scene edge (Chamfer matching). A perfect alignment costs ~0 regardless of how the +shape is filled or shaded. + +It reuses ``visual_match``'s gray loaders / resize / NMS / ``Match`` and ``edge_lines``'s Canny +default, so no matching or geometry code is duplicated. The ``haystack`` is injectable +(ndarray / path / PIL); the search is unit-testable on synthetic arrays. OpenCV + NumPy are +imported lazily. Imports no ``PySide6``. +""" +from typing import Any, List, Optional, Sequence, Tuple + +from je_auto_control.utils.visual_match.visual_match import ( + Match, _haystack_gray, _nms, _resize, _to_gray, +) + +ImageSource = Any +_DEFAULT_CANNY = (50, 150) + + +def _edges(gray, canny: Sequence[int]): + import cv2 + return cv2.Canny(gray, int(canny[0]), int(canny[1])) + + +def _chamfer_score_map(template_gray, scene_gray, canny: Sequence[int]): + """Return ``(score_map, (w, h))`` where higher = better edge alignment, or ``(None, _)``.""" + import cv2 + import numpy as np + scene_edges = _edges(scene_gray, canny) + template_edges = _edges(template_gray, canny) + edge_count = int(np.count_nonzero(template_edges)) + if edge_count == 0 or template_edges.shape[0] > scene_edges.shape[0] \ + or template_edges.shape[1] > scene_edges.shape[1]: + return None, (0, 0) + distance = cv2.distanceTransform(cv2.bitwise_not(scene_edges), + cv2.DIST_L2, 3).astype(np.float32) + mask = (template_edges > 0).astype(np.float32) + total = cv2.matchTemplate(distance, mask, cv2.TM_CCORR) + score_map = 1.0 / (1.0 + total / edge_count) + return score_map, (template_edges.shape[1], template_edges.shape[0]) + + +def _best(score_map, size: Tuple[int, int], scale: float, min_score: float, + current: Optional[Match]) -> Optional[Match]: + import cv2 + _, max_val, _, max_loc = cv2.minMaxLoc(score_map) + if max_val >= min_score and (current is None or max_val > current.score): + return Match(int(max_loc[0]), int(max_loc[1]), size[0], size[1], + round(float(max_val), 4), float(scale)) + return current + + +def edge_match(template: ImageSource, *, haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + scales: Sequence[float] = (1.0,), + canny: Sequence[int] = _DEFAULT_CANNY, + min_score: float = 0.7) -> Optional[Match]: + """Return the best edge-shape (Chamfer) match at or above ``min_score``, or ``None``.""" + template_gray = _to_gray(template) + scene_gray = _haystack_gray(haystack, region) + best: Optional[Match] = None + for scale in scales: + score_map, size = _chamfer_score_map(_resize(template_gray, float(scale)), + scene_gray, canny) + if score_map is not None: + best = _best(score_map, size, float(scale), float(min_score), best) + return best + + +def edge_match_all(template: ImageSource, *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + canny: Sequence[int] = _DEFAULT_CANNY, min_score: float = 0.7, + max_results: int = 20, nms_iou: float = 0.3) -> List[Match]: + """Return every edge-shape match >= ``min_score`` (scale 1.0), overlaps removed (NMS).""" + import numpy as np + score_map, size = _chamfer_score_map(_to_gray(template), + _haystack_gray(haystack, region), canny) + if score_map is None: + return [] + ys, xs = np.nonzero(score_map >= float(min_score)) + candidates = [Match(int(x), int(y), size[0], size[1], + round(float(score_map[y, x]), 4), 1.0) + for y, x in zip(ys, xs)] + return _nms(candidates, float(nms_iou))[:int(max_results)] + + +def chamfer_distance(template: ImageSource, *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + canny: Sequence[int] = _DEFAULT_CANNY) -> float: + """Return the mean edge-to-edge distance at the best alignment (0 = perfect).""" + import cv2 + score_map, _ = _chamfer_score_map(_to_gray(template), + _haystack_gray(haystack, region), canny) + if score_map is None: + return float("inf") + _, max_val, _, _ = cv2.minMaxLoc(score_map) + return round(1.0 / max_val - 1.0, 4) if max_val > 0 else float("inf") diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 3e81d98f..e5435797 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3362,6 +3362,31 @@ def _match_auto(template: str, floor: Any = 0.5, max_results: Any = 20, return {"count": len(matches), "matches": [m.to_dict() for m in matches]} +def _edge_match(template: str, min_score: Any = 0.7, scales: Any = None, + region: Any = None) -> Dict[str, Any]: + """Adapter: best edge-shape (Chamfer) template match on the screen.""" + import json + from je_auto_control.utils.edge_match import edge_match + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + match = edge_match(template, region=region, + scales=_seq_arg(scales, (1.0,)), min_score=float(min_score)) + return {"found": match is not None, + "match": match.to_dict() if match else None} + + +def _edge_match_all(template: str, min_score: Any = 0.7, max_results: Any = 20, + nms_iou: Any = 0.3, region: Any = None) -> Dict[str, Any]: + """Adapter: every edge-shape (Chamfer) match on the screen (NMS).""" + import json + from je_auto_control.utils.edge_match import edge_match_all + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + matches = edge_match_all(template, region=region, min_score=float(min_score), + max_results=int(max_results), nms_iou=float(nms_iou)) + return {"count": len(matches), "matches": [m.to_dict() for m in matches]} + + def _region_arg(value: Any) -> Optional[List[int]]: """Coerce a JSON-string / list region arg into a list of ints, or None.""" import json @@ -5935,6 +5960,8 @@ def __init__(self): "AC_match_with_trust": _match_with_trust, "AC_auto_threshold": _auto_threshold, "AC_match_auto": _match_auto, + "AC_edge_match": _edge_match, + "AC_edge_match_all": _edge_match_all, "AC_grid_cells": _grid_cells, "AC_cell_for_point": _cell_for_point, "AC_point_for_cell": _point_for_cell, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 3fa367fe..74ac12e1 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3698,6 +3698,37 @@ def rotated_match_tools() -> List[MCPTool]: handler=h.match_auto, annotations=READ_ONLY, ), + MCPTool( + name="ac_edge_match", + description=("Find 'template' by its EDGE SHAPE (Chamfer / distance " + "transform), robust to fill / gradient / theme / " + "anti-aliasing where intensity NCC fails and to flat icons " + "ORB can't key on. Returns {found, match}. 'min_score', " + "'scales', 'region'."), + input_schema=schema({ + "template": {"type": "string"}, + "min_score": {"type": "number"}, + "scales": {"type": "array", "items": {"type": "number"}}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["template"]), + handler=h.edge_match, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_edge_match_all", + description=("Find EVERY edge-shape (Chamfer) match of 'template' >= " + "'min_score', overlaps removed by NMS. Returns " + "{count, matches}."), + input_schema=schema({ + "template": {"type": "string"}, + "min_score": {"type": "number"}, + "max_results": {"type": "integer"}, + "nms_iou": {"type": "number"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["template"]), + handler=h.edge_match_all, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 3a9d93ad..a51fbd6b 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2126,6 +2126,17 @@ def match_auto(template, floor=0.5, max_results=20, region=None, return _match_auto(template, floor, max_results, region, method) +def edge_match(template, min_score=0.7, scales=None, region=None): + from je_auto_control.utils.executor.action_executor import _edge_match + return _edge_match(template, min_score, scales, region) + + +def edge_match_all(template, min_score=0.7, max_results=20, nms_iou=0.3, + region=None): + from je_auto_control.utils.executor.action_executor import _edge_match_all + return _edge_match_all(template, min_score, max_results, nms_iou, region) + + def grid_cells(rows, cols, region=None): from je_auto_control.utils.executor.action_executor import _grid_cells return _grid_cells(rows, cols, region) diff --git a/test/unit_test/headless/test_edge_match_batch.py b/test/unit_test/headless/test_edge_match_batch.py new file mode 100644 index 00000000..0e7d3f23 --- /dev/null +++ b/test/unit_test/headless/test_edge_match_batch.py @@ -0,0 +1,66 @@ +"""Headless tests for edge-shape (Chamfer) template matching.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +cv2 = pytest.importorskip("cv2") + +from je_auto_control.utils.edge_match import ( # noqa: E402 + chamfer_distance, edge_match, edge_match_all, +) + + +def _scene(): + hay = np.zeros((200, 200), dtype=np.uint8) + cv2.rectangle(hay, (50, 50), (110, 90), 255, 3) + return hay + + +def _template_different_fill(): + # same outline as the scene rectangle, but drawn at a different grey level + tmpl = np.zeros((50, 70), dtype=np.uint8) + cv2.rectangle(tmpl, (5, 5), (65, 45), 150, 3) + return tmpl + + +def test_finds_shape_despite_different_fill(): + match = edge_match(_template_different_fill(), haystack=_scene(), min_score=0.5) + assert match is not None + assert match.score >= 0.9 # edges align even at a different grey + assert abs(match.x - 45) <= 2 and abs(match.y - 45) <= 2 + + +def test_chamfer_distance_near_zero_on_alignment(): + dist = chamfer_distance(_template_different_fill(), haystack=_scene()) + assert dist < 1.0 # ~0 = edges coincide + + +def test_absent_returns_none(): + blank = np.zeros((200, 200), dtype=np.uint8) + assert edge_match(_template_different_fill(), haystack=blank, + min_score=0.5) is None + + +def test_edge_match_all_dedupes(): + hits = edge_match_all(_template_different_fill(), haystack=_scene(), + min_score=0.85) + assert len(hits) == 1 + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_edge_match", "AC_edge_match_all"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_edge_match", "ac_edge_match_all"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_edge_match", "AC_edge_match_all"} <= specs + + +def test_facade_exports(): + for name in ("edge_match", "edge_match_all", "chamfer_distance"): + assert hasattr(ac, name) and name in ac.__all__