From 67f7884925767c0624ba88174411048725af19e2 Mon Sep 17 00:00:00 2001 From: Peter Jacobson Date: Wed, 10 Jun 2026 10:53:05 +1000 Subject: [PATCH 1/2] Stage browse loading for network drives and 1000-image folders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the synchronous GUI-thread file I/O around the (already staged) thumbnail queue, which froze browsing on networked drives: - Card click previously ran TWO full load_scan calls on the GUI thread (channel previews + a second one just for the header metadata table). New ChannelPreviewLoader reads the scan once on a pool worker and emits meta_ready (plane names + header) then per-plane QImages; the panel builds its slots, previews, and metadata table from those signals. - Opening a viewer previously full-loaded the scan a third time just to map the thumbnail channel to a plane index; now a header-only read_scan_metadata call. - _navigate ran index_folder_shallow synchronously; a cold network folder froze the UI for the whole index. New FolderIndexLoader runs it on a pool worker with token-guarded delivery; the path strip shows "Indexing …" and the previous grid stays interactive meanwhile. - _peek_subfolder content-sniffed every file up to two levels deep with no count cap; a file budget (default 400) now bounds the I/O and a counts_capped flag flows through FolderEntry so folder cards show "N+" lower-bound counts. - Card construction is timer-sliced (first batch synchronous, then zero-delay batches of 120) so 1000-entry folders no longer freeze the GUI building widgets; thumbnail scheduling defers entries whose cards are not built yet, and filtered-out cards are now deleteLater'd on navigation instead of lingering until GC. - Thumbnail runnables enter the global pool at low priority so ViewerLoader / ChannelPreviewLoader / folder indexing never queue behind a screenful of slow network thumbnail reads. Also fixes the py3.12 CI failure: conftest gated the per-test forked isolation on hasplugin("forked"), but pytest-forked registers as "pytest_forked" — the GUI-test process isolation was silently inactive everywhere, which is exactly the recycled-wrapper AttributeError CI hit (QGraphicsItemGroup wrapper reused inside QMenu.addAction). The check now matches the real plugin name (kept in-process on macOS, where fork() under AppKit is unsafe). Co-Authored-By: Claude Fable 5 --- probeflow/core/indexing.py | 20 +++- probeflow/gui/browse/__init__.py | 1 + probeflow/gui/browse/cards.py | 7 +- probeflow/gui/browse/panels.py | 93 +++++++++------- probeflow/gui/browse/thumbnail_grid.py | 147 ++++++++++++++++++++----- probeflow/gui/models.py | 3 + probeflow/gui/workers.py | 93 ++++++++++++++++ tests/conftest.py | 10 +- tests/test_folder_index_shallow.py | 25 +++++ tests/test_gui_index_integration.py | 4 +- 10 files changed, 327 insertions(+), 76 deletions(-) diff --git a/probeflow/core/indexing.py b/probeflow/core/indexing.py index 91ee96a..c19822f 100644 --- a/probeflow/core/indexing.py +++ b/probeflow/core/indexing.py @@ -306,6 +306,7 @@ class SubfolderEntry: n_scans: int # scan files found within peek depth n_specs: int # spectroscopy files found within peek depth sample_scan_paths: tuple[Path, ...] # up to 3 paths for preview thumbnails + counts_capped: bool = False # peek budget hit — counts are lower bounds @dataclass(frozen=True) @@ -322,18 +323,26 @@ def _peek_subfolder( *, max_samples: int = 3, peek_depth: int = 2, + max_files: int = 400, ) -> SubfolderEntry: """Briefly scan *folder* (BFS, capped by peek_depth) for counts and samples. - Bounded so users get a meaningful preview even for nested experiment trees, - without paying for a full recursive walk. + Bounded two ways so users get a meaningful preview even for nested + experiment trees: by depth, and by a *file budget* (``max_files``). Each + recognised-suffix file costs a content sniff — an ~8 KB read — so without + the budget, peeking the parent of a tree whose subfolders hold thousands + of scans transfers megabytes per folder card on a network drive. When the + budget runs out the walk stops and ``counts_capped`` marks the counts as + lower bounds (the grid shows "N+"). """ n_scans = 0 n_specs = 0 + files_examined = 0 + capped = False samples: list[Path] = [] queue: list[tuple[Path, int]] = [(folder, 0)] - while queue: + while queue and not capped: current, depth = queue.pop(0) try: # os.scandir serves is_file()/is_dir() from the single directory @@ -351,6 +360,10 @@ def _peek_subfolder( except OSError: continue if is_file: + if files_examined >= max_files: + capped = True + break + files_examined += 1 p = Path(e.path) ft = sniff_file_type(p) if ft in (FileType.CREATEC_IMAGE, FileType.NANONIS_IMAGE): @@ -368,6 +381,7 @@ def _peek_subfolder( n_scans=n_scans, n_specs=n_specs, sample_scan_paths=tuple(samples), + counts_capped=capped, ) diff --git a/probeflow/gui/browse/__init__.py b/probeflow/gui/browse/__init__.py index 8bb4e07..67ba559 100644 --- a/probeflow/gui/browse/__init__.py +++ b/probeflow/gui/browse/__init__.py @@ -5,6 +5,7 @@ from probeflow.core.scan_loader import load_scan from probeflow.gui.workers import ( ChannelLoader, + ChannelPreviewLoader, ChannelSignals, FolderThumbnailLoader, SpecThumbnailLoader, diff --git a/probeflow/gui/browse/cards.py b/probeflow/gui/browse/cards.py index f05a5e1..aafa2db 100644 --- a/probeflow/gui/browse/cards.py +++ b/probeflow/gui/browse/cards.py @@ -195,11 +195,14 @@ class FolderCard(_BrowseCard): THUMB_H = 56 def __init__(self, entry: FolderEntry, t: dict, parent=None): + # A "+" marks counts that hit the indexing peek's file budget — they + # are lower bounds, not exact totals (big network trees). + plus = "+" if getattr(entry, "counts_capped", False) else "" meta_parts = [] if entry.n_scans: - meta_parts.append(f"{entry.n_scans} scan{'s' if entry.n_scans != 1 else ''}") + meta_parts.append(f"{entry.n_scans}{plus} scan{'s' if entry.n_scans != 1 else ''}") if entry.n_specs: - meta_parts.append(f"{entry.n_specs} spec{'s' if entry.n_specs != 1 else ''}") + meta_parts.append(f"{entry.n_specs}{plus} spec{'s' if entry.n_specs != 1 else ''}") meta = " | ".join(meta_parts) if meta_parts else "(empty)" super().__init__(entry, t, meta, parent=parent) diff --git a/probeflow/gui/browse/panels.py b/probeflow/gui/browse/panels.py index a53aa0c..b83562e 100644 --- a/probeflow/gui/browse/panels.py +++ b/probeflow/gui/browse/panels.py @@ -26,7 +26,6 @@ QWidget, ) -from probeflow.core.scan_loader import load_scan as _default_load_scan from probeflow.gui.models import PLANE_NAMES, SxmFile, VertFile from probeflow.gui.rendering import ( CMAP_KEY, @@ -36,7 +35,7 @@ THUMBNAIL_CHANNEL_DEFAULT, THUMBNAIL_CHANNEL_OPTIONS, ) -from probeflow.gui.workers import ChannelLoader, ChannelSignals +from probeflow.gui.workers import ChannelPreviewLoader from .helpers import _browse_attr, _sep @@ -400,8 +399,9 @@ def show_entry(self, entry: SxmFile, colormap_key: str, self._qi["size"].setText(f"{entry.scan_nm:.1f} nm" if entry.scan_nm is not None else "—") self._qi["bias"].setText(f"{entry.bias_mv:.0f} mV" if entry.bias_mv is not None else "—") self._qi["setp"].setText(_setp_display(entry)) + # One worker load serves both the channel previews and the metadata + # table (meta_ready → _populate_metadata_rows). self.load_channels(entry, colormap_key, processing=None) - self._load_metadata(entry) def show_vert_entry(self, entry: VertFile): self.name_lbl.setText(entry.stem) @@ -439,6 +439,9 @@ def _load_vert_metadata(self, entry: VertFile): def clear(self): self.name_lbl.setText("No scan selected") + # Invalidate any in-flight preview load so late results don't + # repopulate a panel the user just cleared. + self._ch_token = object() for v in self._qi.values(): v.setText("—") for lbl in self._ch_img_lbls: @@ -453,43 +456,51 @@ def apply_theme(self, t: dict): # ── Public ───────────────────────────────────────────────────────────────── def load_channels(self, entry: SxmFile, colormap_key: str, processing: dict = None): + """Kick off the off-thread preview + header load for *entry*. + + The scan is read exactly once, on a pool worker (previously this did a + synchronous full ``load_scan`` here for the previews and a second one + in the metadata loader — two full file transfers on the GUI thread per + card click, the main network-drive freeze). ``meta_ready`` populates + the slot layout and metadata table; ``loaded`` fills in each preview. + Stale deliveries are filtered by the token check in each slot. + """ self._ch_token = object() - # One signals object for the panel's lifetime. Re-creating it per call - # left the previous instance solely owned by still-running ChannelLoader - # runnables; QThreadPool auto-deletes those on the worker thread, which - # C++-destroys the signals QObject (and its cross-thread connections) - # off the main thread — the SIGSEGV class documented on _PooledWorker. - # Stale deliveries are filtered by the token check in _on_ch_loaded. - sigs = getattr(self, "_ch_sigs", None) - if sigs is None: - Signals = _browse_attr("ChannelSignals", ChannelSignals) - sigs = Signals() - sigs.setParent(self) - sigs.loaded.connect(self._on_ch_loaded) - self._ch_sigs = sigs - planes = [] - try: - scan = _browse_attr("load_scan", _default_load_scan)(entry.path) - plane_names = list(scan.plane_names) - n_planes = scan.n_planes - planes = list(getattr(scan, "planes", []) or []) - except Exception: - plane_names = list(PLANE_NAMES) - n_planes = len(plane_names) - self._set_channel_preview_slots(plane_names) - for i in range(n_planes): - arr = planes[i] if i < len(planes) else None - Loader = _browse_attr("ChannelLoader", ChannelLoader) - loader = Loader(entry, i, colormap_key, - self._ch_token, 124, 98, sigs, - self._clip_low, self._clip_high, - processing=processing, - arr=arr) - self._pool.start(loader) + self._ch_entry = entry + for lbl in self._ch_img_lbls: + lbl.clear() + lbl.setText("…") + Loader = _browse_attr("ChannelPreviewLoader", ChannelPreviewLoader) + loader = Loader(entry, colormap_key, self._ch_token, 124, 98, + self._clip_low, self._clip_high, + processing=processing) + loader.signals.meta_ready.connect(self._on_ch_meta) + loader.signals.loaded.connect(self._on_ch_loaded) + loader.signals.failed.connect(self._on_ch_failed) + self._pool.start(loader) # Back-compat alias used internally _load_channels = load_channels + @Slot(list, dict, object) + def _on_ch_meta(self, names: list, header: dict, token) -> None: + if token is not self._ch_token: + return + self._set_channel_preview_slots(list(names)) + entry = getattr(self, "_ch_entry", None) + if entry is not None: + self._populate_metadata_rows(entry, dict(header)) + + @Slot(str, object) + def _on_ch_failed(self, message: str, token) -> None: + if token is not self._ch_token: + return + for lbl in self._ch_img_lbls: + lbl.clear() + lbl.setText("load failed") + self._meta_rows = [] + self._filter_meta() + @Slot(int, QImage, object) def _on_ch_loaded(self, idx: int, image: QImage, token): # Workers emit QImage (QPixmap is GUI-thread-only); convert here. @@ -540,11 +551,13 @@ def _set_channel_preview_slots(self, names: list[str]) -> None: self._ch_img_lbls.append(img_lbl) self._ch_name_lbls.append(nm_lbl) - def _load_metadata(self, entry: SxmFile): - try: - hdr = _browse_attr("load_scan", _default_load_scan)(entry.path).header - except Exception: - hdr = {} + def _populate_metadata_rows(self, entry: SxmFile, hdr: dict) -> None: + """Fill the metadata table from an already-loaded header dict. + + The header arrives via ``ChannelPreviewLoader.meta_ready`` so the file + is read once, off the GUI thread (this used to be a second synchronous + full ``load_scan`` per card click). + """ priority = [ "REC_DATE", "REC_TIME", "SCAN_PIXELS", "SCAN_RANGE", "SCAN_OFFSET", "SCAN_ANGLE", "SCAN_DIR", "BIAS", diff --git a/probeflow/gui/browse/thumbnail_grid.py b/probeflow/gui/browse/thumbnail_grid.py index a6baba6..ccd14c8 100644 --- a/probeflow/gui/browse/thumbnail_grid.py +++ b/probeflow/gui/browse/thumbnail_grid.py @@ -10,7 +10,6 @@ from PySide6.QtGui import QCursor, QImage, QPixmap from PySide6.QtWidgets import QFrame, QGridLayout, QHBoxLayout, QLabel, QPushButton, QScrollArea, QVBoxLayout, QWidget -from probeflow.core.scan_loader import load_scan as _default_load_scan from probeflow.gui.models import FolderEntry, SxmFile, VertFile, browse_entry_key from probeflow.gui.rendering import ( DEFAULT_CMAP_KEY, @@ -18,12 +17,25 @@ THUMBNAIL_CHANNEL_OPTIONS, resolve_thumbnail_plane_index, ) -from probeflow.gui.workers import FolderThumbnailLoader, SpecThumbnailLoader, ThumbnailLoader +from probeflow.gui.workers import ( + FolderIndexLoader, + FolderThumbnailLoader, + SpecThumbnailLoader, + ThumbnailLoader, +) from .breadcrumbs import _BreadcrumbBar from .cards import FolderCard, ScanCard, SpecCard, _BrowseCard from .helpers import _browse_attr, _CARD_SIZE_PRESETS, _is_deleted_qt_runtime_error +# Thumbnail renders share QThreadPool.globalInstance() with interactive loads +# (ViewerLoader, ChannelPreviewLoader, _ScanLoadWorker). Queue them below the +# default priority (0) so opening a viewer never waits behind a screenful of +# queued thumbnail reads on a slow network folder. Already-running thumbnail +# workers still finish, but everything queued yields. +_THUMBNAIL_PRIORITY = -1 + + # ── ThumbnailGrid ───────────────────────────────────────────────────────────── class ThumbnailGrid(QWidget): """ @@ -108,11 +120,15 @@ def __init__(self, t: dict, parent=None): self._thumbnail_clip: tuple[float, float] = (1.0, 99.0) self._thumbnail_channel: str = THUMBNAIL_CHANNEL_DEFAULT self._load_token = object() + self._nav_token = object() self._current_cols: int = 1 self._filter_mode: str = "all" self._thumbnail_size_name: str = "large" self._thumbnail_pending: dict[str, Union[SxmFile, VertFile, FolderEntry]] = {} self._background_thumbnail_batch_size: int = 2 + # Timer-sliced card construction state (see _build_card_batch). + self._card_build_queue: list = [] + self._next_grid_index: int = 0 self._visible_thumb_timer = QTimer(self) self._visible_thumb_timer.setSingleShot(True) @@ -170,15 +186,39 @@ def refresh(self) -> None: self._navigate(self._current_dir) def _navigate(self, path: Path): - """Index *path* shallowly and rebuild the grid + breadcrumb.""" - from probeflow.core.indexing import index_folder_shallow + """Index *path* shallowly off-thread, then rebuild the grid. + + The breadcrumb and path strip update immediately (navigation intent); + the current grid stays interactive until the index arrives. Cold + network folders previously froze the GUI here for the whole index. + """ + path = Path(path) + self._current_dir = path + if self._root is None: + self._root = path + self._breadcrumb.set_state( + self._root, self._current_dir, + can_go_back=bool(self._history), + ) + self._refresh_btn.setEnabled(False) # re-enabled when the index lands + self._path_lbl.setText(f"Indexing {path.name}…") + self._nav_token = object() + loader = FolderIndexLoader(path, self._nav_token) + loader.signals.indexed.connect(self._on_folder_indexed) + loader.signals.failed.connect(self._on_folder_index_failed) + # Default (interactive) priority — must not queue behind thumbnails. + self._pool.start(loader) + + @Slot(object, object, object) + def _on_folder_indexed(self, path, index, token): + if token is not getattr(self, "_nav_token", None): + return from probeflow.gui.models import ( FolderEntry as _FE, _scan_items_to_sxm, _spec_items_to_vert, ) - index = index_folder_shallow(path, include_errors=True) scan_items = [it for it in index.files if it.item_type == "scan"] spec_items = [it for it in index.files if it.item_type == "spectrum"] sxm_entries = _scan_items_to_sxm(scan_items) @@ -190,17 +230,17 @@ def _navigate(self, path: Path): file_entries = sorted(sxm_entries + vert_entries, key=lambda e: e.stem) entries: list[Union[SxmFile, VertFile, FolderEntry]] = list(folder_entries) + list(file_entries) - self._current_dir = path - if self._root is None: - self._root = path - self._breadcrumb.set_state( - self._root, self._current_dir, - can_go_back=bool(self._history), - ) self._refresh_btn.setEnabled(True) self._render_entries(entries) self.folder_changed.emit(path) + @Slot(object, str, object) + def _on_folder_index_failed(self, path, message, token): + if token is not getattr(self, "_nav_token", None): + return + self._refresh_btn.setEnabled(True) + self._path_lbl.setText(f"Could not open {Path(path).name}: {message}") + def load(self, entries: list, folder_path: str = ""): """Legacy entry point: render a flat list of entries (no navigation). @@ -238,12 +278,19 @@ def _render_entries(self, entries: list): f"({', '.join(parts) if parts else '0 items'})" ) - # clear grid + # clear grid (gridded widgets), then any filtered-out cards that were + # never re-added to the layout — those are parentless and would + # otherwise linger until garbage collection. while self._grid.count(): item = self._grid.takeAt(0) if item.widget(): item.widget().deleteLater() + for card in self._cards.values(): + if card.parent() is None: + card.deleteLater() self._cards = {} + self._card_build_queue = [] + self._next_grid_index = 0 if not entries: self._empty_lbl = QLabel("No scans, spectra, or subfolders here") @@ -252,9 +299,30 @@ def _render_entries(self, entries: list): self._grid.addWidget(self._empty_lbl, 0, 0) return - cols = self._calc_cols() - self._current_cols = cols - for entry in entries: + self._current_cols = self._calc_cols() + # Build the first batch synchronously so the folder appears instantly, + # then drain the rest in zero-delay timer slices: constructing ~1000 + # card widgets in one loop froze the GUI for seconds, independent of + # disk speed. The token ties the slices to this navigation. + self._card_build_queue = list(entries) + self._build_card_batch(self._load_token) + + # Queue all thumbnails now; scheduling skips entries whose cards are + # not built yet and picks them up on later refresh ticks. + self._prepare_thumbnail_queue(entries) + + # How many cards to construct per slice. ~120 keeps each slice well under + # a frame budget on typical hardware while finishing 1000 entries in <10 + # event-loop turns. + _CARD_BUILD_BATCH = 120 + + def _build_card_batch(self, token) -> None: + if token is not self._load_token or not self._card_build_queue: + return + batch = self._card_build_queue[: self._CARD_BUILD_BATCH] + del self._card_build_queue[: self._CARD_BUILD_BATCH] + cols = self._current_cols + for entry in batch: key = self._key_for(entry) if isinstance(entry, FolderEntry): card = FolderCard(entry, self._t) @@ -269,11 +337,21 @@ def _render_entries(self, entries: list): if self._thumbnail_size_name == "small": card.set_compact_mode(True) self._cards[key] = card - - # Populate the grid honouring the current filter. - self._relayout_filtered() - - self._prepare_thumbnail_queue(entries) + # Append in entry order honouring the current filter; a filter or + # column change mid-build triggers _relayout_filtered, which + # re-places built cards and resets _next_grid_index. + if self._is_entry_visible(entry): + row, col = divmod(self._next_grid_index, cols) + self._grid.addWidget(card, row, col, Qt.AlignTop | Qt.AlignLeft) + card.setVisible(True) + self._next_grid_index += 1 + else: + card.setVisible(False) + if self._card_build_queue: + QTimer.singleShot( + 0, self, lambda tok=token: self._build_card_batch(tok) + ) + self._schedule_visible_thumbnail_refresh(delay_ms=0) @staticmethod def _key_for(entry) -> str: @@ -375,9 +453,15 @@ def _visible_thumbnail_keys(self) -> list[str]: return keys def _start_thumbnail_for_key(self, key: str) -> bool: - entry = self._thumbnail_pending.pop(key, None) + entry = self._thumbnail_pending.get(key) if entry is None: return False + # Card not constructed yet (timer-sliced build still draining): keep + # the entry pending so the result has somewhere to land; a later + # refresh tick retries. + if key not in self._cards: + return False + self._thumbnail_pending.pop(key, None) token = self._load_token if isinstance(entry, FolderEntry): Loader = _browse_attr("FolderThumbnailLoader", FolderThumbnailLoader) @@ -391,18 +475,18 @@ def _start_thumbnail_for_key(self, key: str) -> bool: thumbnail_channel=self._thumbnail_channel, ) loader.signals.loaded.connect(self._on_folder_thumbs) - self._pool.start(loader) + self._pool.start(loader, _THUMBNAIL_PRIORITY) return True if isinstance(entry, VertFile): Loader = _browse_attr("SpecThumbnailLoader", SpecThumbnailLoader) loader = Loader(entry, token, SpecCard.IMG_W, SpecCard.IMG_H) loader.signals.loaded.connect(self._on_thumb) - self._pool.start(loader) + self._pool.start(loader, _THUMBNAIL_PRIORITY) return True if isinstance(entry, SxmFile): loader = self._make_thumbnail_loader(entry, token) loader.signals.loaded.connect(self._on_thumb) - self._pool.start(loader) + self._pool.start(loader, _THUMBNAIL_PRIORITY) return True return False @@ -507,13 +591,17 @@ def set_entry_processing(self, path_str: str, proc: dict) -> None: self._thumbnail_pending.pop(self._key_for(entry), None) loader = self._make_thumbnail_loader(entry, token) loader.signals.loaded.connect(self._on_thumb) - self._pool.start(loader) + self._pool.start(loader, _THUMBNAIL_PRIORITY) break def thumbnail_plane_index_for_entry(self, entry: SxmFile) -> int: + # Header-only metadata read: this runs on the GUI thread when a viewer + # is opened, and a full load_scan here meant transferring the entire + # file (then the viewer transferred it again) before the window showed. try: - scan = _browse_attr("load_scan", _default_load_scan)(entry.path) - return resolve_thumbnail_plane_index(scan.plane_names, self._thumbnail_channel) + from probeflow.core.metadata import read_scan_metadata + names = read_scan_metadata(entry.path).plane_names + return resolve_thumbnail_plane_index(list(names), self._thumbnail_channel) except Exception: return 0 @@ -709,6 +797,9 @@ def _relayout_filtered(self): self._grid.addWidget(card, row, col, Qt.AlignTop | Qt.AlignLeft) card.setVisible(True) i += 1 + # Keep incremental (timer-sliced) card placement appending after the + # cards this full pass just laid out. + self._next_grid_index = i def set_thumbnail_size(self, name: str) -> None: """Switch all cards to a size preset ("large" or "small").""" diff --git a/probeflow/gui/models.py b/probeflow/gui/models.py index 4ec1576..b35f3ca 100644 --- a/probeflow/gui/models.py +++ b/probeflow/gui/models.py @@ -106,6 +106,8 @@ class FolderEntry: n_scans: int = 0 n_specs: int = 0 sample_scan_paths: tuple[Path, ...] = field(default_factory=tuple) + # The indexing peek stopped at its file budget — counts are lower bounds. + counts_capped: bool = False @property def stem(self) -> str: @@ -118,6 +120,7 @@ def from_index(cls, sub) -> "FolderEntry": n_scans=sub.n_scans, n_specs=sub.n_specs, sample_scan_paths=tuple(sub.sample_scan_paths), + counts_capped=bool(getattr(sub, "counts_capped", False)), ) diff --git a/probeflow/gui/workers.py b/probeflow/gui/workers.py index 6719c83..ff60ced 100644 --- a/probeflow/gui/workers.py +++ b/probeflow/gui/workers.py @@ -251,6 +251,67 @@ def work(self): self.signals.loaded.emit(browse_entry_key(self.entry), QImage(), self.token) +# ── Worker: browse channel previews + header (single off-thread load) ───────── +class ChannelPreviewSignals(QObject): + meta_ready = Signal(list, dict, object) # plane_names, header, token + loaded = Signal(int, QImage, object) # plane idx, image, token + failed = Signal(str, object) # message, token + + +class ChannelPreviewLoader(_PooledWorker): + """Load a scan once off the GUI thread and render its channel previews. + + Replaces the browse info panel's synchronous double load — one full + ``load_scan`` for the channel thumbnails plus a second one just for the + header metadata table — which froze the GUI for two full file transfers + per card click on a network drive. Emits ``meta_ready`` (plane names + + header) first so the panel can build its preview slots and metadata table, + then one ``loaded`` per plane. + """ + + def __init__(self, entry: SxmFile, colormap: str, token, w: int, h: int, + clip_low: float = 1.0, clip_high: float = 99.0, + processing: dict = None): + super().__init__(ChannelPreviewSignals()) + self.entry = entry + self.colormap = colormap + self.token = token + self.w = w + self.h = h + self.clip_low = clip_low + self.clip_high = clip_high + self.processing = processing or {} + + def work(self): + from probeflow.core.scan_loader import load_scan + + try: + scan = load_scan(self.entry.path) + except Exception as exc: + _log_preview_failure("ChannelPreviewLoader", "load", self.entry.path, exc) + self.signals.failed.emit(str(exc), self.token) + return + names = list(scan.plane_names or []) or [ + f"Channel {i}" for i in range(scan.n_planes) + ] + header = dict(getattr(scan, "header", {}) or {}) + self.signals.meta_ready.emit(names, header, self.token) + for i in range(scan.n_planes): + img = render_scan_image( + arr=scan.planes[i], + colormap=self.colormap, + clip_low=self.clip_low, + clip_high=self.clip_high, + size=(self.w, self.h), + processing=self.processing or None, + ) + self.signals.loaded.emit( + i, + pil_to_qimage(img) if img is not None else QImage(), + self.token, + ) + + # ── Worker: channel thumbnails ──────────────────────────────────────────────── class ChannelSignals(QObject): loaded = Signal(int, QImage, object) @@ -347,6 +408,38 @@ def work(self): self.signals.failed.emit(f"Image render failed: {exc}", self.token) +# ── Worker: shallow folder index ────────────────────────────────────────────── +class FolderIndexSignals(QObject): + indexed = Signal(object, object, object) # path, ShallowFolderIndex, token + failed = Signal(object, str, object) # path, message, token + + +class FolderIndexLoader(_PooledWorker): + """Run ``index_folder_shallow`` off the GUI thread. + + A cold visit to a large folder on a network drive takes seconds even with + the indexing layer's internal parallelism; doing it synchronously in + ``ThumbnailGrid._navigate`` froze the whole UI for that long. Stale + results (the user navigated again) are dropped via the token. + """ + + def __init__(self, path, token) -> None: + super().__init__(FolderIndexSignals()) + self.path = Path(path) + self.token = token + + def work(self): + from probeflow.core.indexing import index_folder_shallow + + try: + index = index_folder_shallow(self.path, include_errors=True) + except Exception as exc: + _log.warning("FolderIndexLoader: failed to index %s (%s)", self.path, exc) + self.signals.failed.emit(self.path, str(exc), self.token) + return + self.signals.indexed.emit(self.path, index, self.token) + + # ── Worker: conversion ──────────────────────────────────────────────────────── class ConversionSignals(QObject): log_msg = Signal(str, str) diff --git a/tests/conftest.py b/tests/conftest.py index a64a8ac..2642f05 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -107,7 +107,15 @@ def pytest_collection_modifyitems(config, items): # and contains any residual crash to a single reported test instead of # aborting the whole run. Requires pytest-forked; without it the marker is a # harmless no-op and the tests run in-process (the prior behaviour). - if config.pluginmanager.hasplugin("forked"): + # + # pytest-forked registers its plugin under the name "pytest_forked" — the + # bare "forked" check matched nothing, which silently disabled this whole + # mechanism (seen as the recycled-wrapper AttributeError on CI py3.12). + # fork() is unsafe under macOS AppKit, so keep in-process behaviour there. + if sys.platform != "darwin" and ( + config.pluginmanager.hasplugin("pytest_forked") + or config.pluginmanager.hasplugin("forked") + ): for item in gui_items: item.add_marker(pytest.mark.forked) diff --git a/tests/test_folder_index_shallow.py b/tests/test_folder_index_shallow.py index cf672ab..8194ca5 100644 --- a/tests/test_folder_index_shallow.py +++ b/tests/test_folder_index_shallow.py @@ -102,3 +102,28 @@ def test_empty_folder(tmp_path): idx = index_folder_shallow(tmp_path) assert idx.files == [] assert idx.subfolders == [] + + +def test_peek_file_budget_caps_counts(tmp_path): + """_peek_subfolder stops at its file budget and marks counts as capped. + + Each recognised-suffix file costs an ~8 KB content sniff; without the + budget, peeking the parent of a big network tree reads every file. + """ + import shutil + + from probeflow.core.indexing import _peek_subfolder + + src = next(TESTDATA.glob("*.sxm"), None) or next(TESTDATA.glob("*.dat")) + sub = tmp_path / "experiment" + sub.mkdir() + for i in range(10): + shutil.copy(src, sub / f"scan_{i:02d}{src.suffix}") + + capped = _peek_subfolder(sub, max_files=4) + assert capped.counts_capped is True + assert capped.n_scans <= 4 + + uncapped = _peek_subfolder(sub, max_files=400) + assert uncapped.counts_capped is False + assert uncapped.n_scans == 10 diff --git a/tests/test_gui_index_integration.py b/tests/test_gui_index_integration.py index c61502c..b06e6d4 100644 --- a/tests/test_gui_index_integration.py +++ b/tests/test_gui_index_integration.py @@ -406,7 +406,7 @@ def test_thumbnail_grid_prioritizes_visible_cards_before_background(qapp, monkey started = [] - def start(loader): + def start(loader, priority=0): started.append(loader.entry.path.name) grid = ThumbnailGrid(THEMES["dark"]) @@ -448,7 +448,7 @@ class BusyPool: def __init__(self): self.started = [] - def start(self, loader): + def start(self, loader, priority=0): self.started.append(loader) def activeThreadCount(self): From 4d0c41c619cec3e9d97506c10dae04a4883ad8ae Mon Sep 17 00:00:00 2001 From: Peter Jacobson Date: Wed, 10 Jun 2026 11:07:38 +1000 Subject: [PATCH 2/2] Replace broken forked GUI-test isolation with boundary GC drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enabling the forked marker (previous commit) surfaced that pytest-forked (archived, last release 2023) no longer cooperates with modern pytest teardown bookkeeping: every non-forked test following a forked one errors with "previous item was not torn down properly", and the changed import order broke the matplotlib backend test. Remove the forked mechanism entirely (and the pytest-forked dependency) and close the wrapper-recycling window in-process instead: the drain fixture now garbage-collects after each GUI test, so parentless widgets a test leaked are destroyed deterministically at the test boundary — visible to Shiboken — rather than at a random GC point mid-way through a later test, which is what recycled C++ addresses into stale wrappers ("'QGraphicsItemGroup' object has no attribute 'connect'" on CI py3.12). The collect is gated to GUI-module tests to keep the suite fast. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 4 +-- tests/conftest.py | 63 ++++++++++++++++++++++++++++++----------------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a2925b1..9c18a98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,10 +20,10 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest", "pytest-forked", "vulture"] +dev = ["pytest", "vulture"] features = ["opencv-python", "scikit-learn"] gwyddion = ["gwyfile"] -all = ["opencv-python", "scikit-learn", "gwyfile", "pytest", "pytest-forked"] +all = ["opencv-python", "scikit-learn", "gwyfile", "pytest"] [project.scripts] probeflow = "probeflow.cli:main" diff --git a/tests/conftest.py b/tests/conftest.py index 2642f05..3ff4323 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -98,30 +98,34 @@ def pytest_collection_modifyitems(config, items): item.add_marker(marker) return - # Qt works here: run each GUI test in its own forked subprocess. Offscreen - # Qt is a single process-wide QApplication with a global QThreadPool and a - # PySide wrapper cache keyed by C++ pointer address; objects leaked by one - # test get their address (and stale Python wrapper) reused by a later test, - # which surfaces as an intermittent SIGSEGV or a bogus AttributeError on a - # recycled wrapper. A fresh address space per test removes the reuse entirely - # and contains any residual crash to a single reported test instead of - # aborting the whole run. Requires pytest-forked; without it the marker is a - # harmless no-op and the tests run in-process (the prior behaviour). - # - # pytest-forked registers its plugin under the name "pytest_forked" — the - # bare "forked" check matched nothing, which silently disabled this whole - # mechanism (seen as the recycled-wrapper AttributeError on CI py3.12). - # fork() is unsafe under macOS AppKit, so keep in-process behaviour there. - if sys.platform != "darwin" and ( - config.pluginmanager.hasplugin("pytest_forked") - or config.pluginmanager.hasplugin("forked") - ): - for item in gui_items: - item.add_marker(pytest.mark.forked) + # NOTE on process isolation: a previous design forked each GUI test via + # pytest-forked to defeat PySide wrapper-cache recycling (a stale Python + # wrapper at a reused C++ address surfaces as an intermittent SIGSEGV or a + # bogus AttributeError, e.g. "'QGraphicsItemGroup' object has no attribute + # 'connect'" inside QMenu.addAction). The marker was silently inactive — it + # gated on hasplugin("forked") but the plugin registers as "pytest_forked" + # — and actually enabling it showed pytest-forked (archived, last release + # 2023) no longer cooperates with modern pytest teardown bookkeeping: + # every non-forked test following a forked one errors with "previous item + # was not torn down properly". The mechanism is therefore removed; the + # recycling window is instead closed in-process by _drain_qt_between_tests, + # which garbage-collects leaked widgets at the test boundary so their C++ + # objects are destroyed deterministically (and visibly to Shiboken) rather + # than at a random GC point mid-way through a later test. + + +def _is_gui_test(node) -> bool: + """Whether *node* belongs to the modules that create real Qt widgets.""" + module_name = Path(str(getattr(node, "path", getattr(node, "fspath", "")))).name + return ( + module_name in GUI_TEST_MODULES + or module_name in MIXED_QT_FIXTURE_MODULES + or (module_name, node.name) in MIXED_QT_TESTS + ) @pytest.fixture(autouse=True) -def _drain_qt_between_tests(): +def _drain_qt_between_tests(request): """Retire Qt background workers and deferred deletions after each test. The GUI tests start real ``QThreadPool`` workers (e.g. ``ViewerLoader``) and @@ -149,6 +153,8 @@ def _drain_qt_between_tests(): if app is None: return + import gc + from PySide6.QtCore import QEvent, QThreadPool # 1. Let in-flight workers finish so their loaded()/failed() events are all @@ -158,10 +164,21 @@ def _drain_qt_between_tests(): # 2. Deliver queued cross-thread signals to receivers that are still alive # (token-guarded slots drop stale content harmlessly). app.processEvents() - # 3. Now actually destroy everything deleteLater()'d; Qt drops each object's + # 3. Collect leaked Python-owned Qt objects (parentless dialogs/canvases a + # test created and dropped without deleteLater). Without this they are + # destroyed at a nondeterministic GC point inside a *later* test, and + # their recycled C++ addresses resurrect stale Shiboken wrappers — the + # intermittent "'QGraphicsItemGroup' object has no attribute 'connect'" + # class of failure. Collecting here makes the destruction deterministic + # and visible to Shiboken at the test boundary. A full collect costs + # ~100 ms with this suite's heaps, so only GUI tests — the ones that + # actually leak widgets — pay it. + if _is_gui_test(request.node): + gc.collect() + # 4. Now actually destroy everything deleteLater()'d; Qt drops each object's # pending posted events as it runs the destructor. app.sendPostedEvents(None, QEvent.DeferredDelete) - # 4. Settle anything the deletions themselves posted. + # 5. Settle anything the deletions themselves posted. app.processEvents()