Support standard raster image formats

pyproject.toml

@@ -37,9 +37,11 @@ classifiers = [
 ]
 
 dependencies = [
+    "arraybridge>=0.2.9",
     "numpy>=1.26.0",
     "portalocker>=2.8.0",  # Cross-platform file locking
     "metaclass-registry",
+    "imageio>=2.37.0",
     "zarr>=2.18.0,<3.0",  # Required for ZarrStorageBackend
     "ome-zarr>=0.11.0",   # Required for OME-ZARR HCS compliance
 ]
@@ -197,4 +199,4 @@ ignore = [
 ]
 
 [tool.ruff.per-file-ignores]
-"__init__.py" = ["F401"]  # unused imports
+"__init__.py" = ["F401"]  # unused imports

src/polystore/__init__.py

@@ -26,10 +26,10 @@
     get_backend,
 )
 from .constants import Backend, MemoryType, TransportMode
-from .disk import DiskStorageBackend
+from .disk import DiskBackend, DiskStorageBackend
 from .filemanager import FileManager
 from .formats import FileFormat, DEFAULT_IMAGE_EXTENSIONS
-from .memory import MemoryStorageBackend
+from .memory import MemoryBackend, MemoryStorageBackend
 from .metadata_writer import (
     AtomicMetadataWriter,
     MetadataWriteError,
@@ -76,7 +76,9 @@
     "register_cleanup_callback",
     "STORAGE_BACKENDS",
     "DiskStorageBackend",
+    "DiskBackend",
     "MemoryStorageBackend",
+    "MemoryBackend",
     "FileManager",
     "file_lock",
     "atomic_write_json",

src/polystore/base.py

@@ -546,15 +546,16 @@ def reset_memory_backend() -> None:
     # Clear files from existing memory backend while preserving directories
     memory_backend = storage_registry[Backend.MEMORY.value]
 
-    # DEBUG: Log what's in memory before clearing
     existing_keys = list(memory_backend._memory_store.keys())
-    logger.info(f"🔍 VFS_CLEAR: Memory backend has {len(existing_keys)} entries BEFORE clear")
-    logger.info(f"🔍 VFS_CLEAR: First 10 keys: {existing_keys[:10]}")
+    logger.debug("Memory backend has %s entries before clear", len(existing_keys))
+    logger.debug("First memory backend keys before clear: %s", existing_keys[:10])
 
     memory_backend.clear_files_only()
 
-    # DEBUG: Log what's in memory after clearing
     remaining_keys = list(memory_backend._memory_store.keys())
-    logger.info(f"🔍 VFS_CLEAR: Memory backend has {len(remaining_keys)} entries AFTER clear (directories only)")
-    logger.info(f"🔍 VFS_CLEAR: First 10 remaining keys: {remaining_keys[:10]}")
+    logger.debug(
+        "Memory backend has %s entries after clear (directories only)",
+        len(remaining_keys),
+    )
+    logger.debug("First memory backend keys after clear: %s", remaining_keys[:10])
     logger.info("Memory backend reset - files cleared, directories preserved")

src/polystore/disk.py

@@ -9,6 +9,7 @@
 import logging
 import os
 import shutil
+import importlib
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Set, Union
 
@@ -23,7 +24,7 @@
 
 def optional_import(module_name):
     try:
-        return __import__(module_name)
+        return importlib.import_module(module_name)
     except ImportError:
         return None
 
@@ -44,6 +45,7 @@ def optional_import(module_name):
     cupy = get_cupy()
     tf = get_tf()
 tifffile = optional_import("tifffile")
+imageio = optional_import("imageio.v3")
 
 # Optional arraybridge integration for memory conversion
 try:
@@ -99,6 +101,7 @@ def _register_formats(self):
 
             # Complex formats - use custom handlers
             (FileFormat.TIFF, tifffile, self._tiff_writer, self._tiff_reader),
+            (FileFormat.RASTER_IMAGE, imageio, self._image_writer, self._image_reader),
             (FileFormat.TEXT, True, self._text_writer, self._text_reader),
             (FileFormat.JSON, True, self._json_writer, self._json_reader),
             (FileFormat.CSV, True, self._csv_writer, self._csv_reader),
@@ -164,6 +167,14 @@ def _tiff_reader(self, path):
         else:
             return tifffile.imread(str(path))
 
+    def _image_writer(self, path, data, **kwargs):
+        """Write standard raster images using imageio."""
+        imageio.imwrite(path, np.asarray(data))
+
+    def _image_reader(self, path):
+        """Read standard raster images using imageio."""
+        return imageio.imread(path)
+
     def _text_writer(self, path, data, **kwargs):
         """Write text data to file. Accepts and ignores extra kwargs for compatibility."""
         path.write_text(str(data))
@@ -261,7 +272,7 @@ def load(self, file_path: Union[str, Path], **kwargs) -> Any:
             ext = disk_path.suffix.lower()
 
         if not self.format_registry.is_registered(ext):
-            raise ValueError(f"No writer registered for extension '{ext}'")
+            raise ValueError(f"No reader registered for extension '{ext}'")
 
         try:
             reader = self.format_registry.get_reader(ext)
@@ -823,3 +834,6 @@ def _save_rois(self, rois: List, output_path: Path, images_dir: str = None, **kw
 
         logger.info(f"Saved {roi_count} ROIs to .roi.zip archive: {output_path}")
         return str(output_path)
+
+
+DiskBackend = DiskStorageBackend

src/polystore/fiji_stream.py

@@ -31,12 +31,9 @@ class FijiStreamingBackend(StreamingBackend):
     """Fiji streaming backend with ZMQ publisher pattern (matches Napari architecture)."""
     _backend_type = Backend.FIJI_STREAM.value
 
-    # Configure ABC attributes
     VIEWER_TYPE = 'fiji'
     SHM_PREFIX = 'fiji_'
 
-    # __init__, _get_publisher, save, cleanup now inherited from ABC
-
     def _prepare_rois_data(self, data: Any, file_path: Union[str, Path]) -> dict:
         """
         Prepare ROIs data for transmission.
@@ -90,6 +87,7 @@ def save_batch(self, data_list: List[Any], file_paths: List[Union[str, Path]], *
         source = kwargs.get('source', 'unknown_source')  # Pre-built source value
         images_dir = kwargs.get('images_dir')  # Source image subdirectory for ROI mapping
         plate_path = kwargs.get('plate_path')
+        component_metadata = kwargs.get('component_metadata')
         logger.info(f"🏷️  FIJI BACKEND: plate_path = {plate_path}")
         logger.info(f"🏷️  FIJI BACKEND: microscope_handler = {microscope_handler}")
         display_payload_extra = {
@@ -108,6 +106,7 @@ def save_batch(self, data_list: List[Any], file_paths: List[Union[str, Path]], *
             display_config,
             self._prepare_batch_item,
             plate_path=plate_path,
+            component_metadata=component_metadata,
             component_names_kwargs={"log_prefix": "🏷️  FIJI BACKEND", "verbose": True},
             display_payload_extra=display_payload_extra,
             message_extra=message_extra,

src/polystore/formats.py

@@ -20,6 +20,7 @@ class FileFormat(Enum):
 
     # Image formats
     TIFF = "tiff"
+    RASTER_IMAGE = "raster_image"
 
     # Data formats
     CSV = "csv"
@@ -44,14 +45,22 @@ def extensions(self):
     FileFormat.TENSORFLOW: [".tf"],
     FileFormat.ZARR: [".zarr"],
     FileFormat.TIFF: [".tif", ".tiff"],
+    FileFormat.RASTER_IMAGE: [".bmp", ".gif", ".jpeg", ".jpg", ".png"],
     FileFormat.CSV: [".csv"],
     FileFormat.JSON: [".json"],
     FileFormat.TEXT: [".txt"],
     FileFormat.ROI: [".roi.zip"],
 }
 
 # Default image extensions
-DEFAULT_IMAGE_EXTENSIONS = {".tif", ".tiff", ".TIF", ".TIFF"}
+DEFAULT_IMAGE_EXTENSIONS = {
+    extension
+    for extensions in (
+        FILE_FORMAT_EXTENSIONS[FileFormat.TIFF],
+        FILE_FORMAT_EXTENSIONS[FileFormat.RASTER_IMAGE],
+    )
+    for extension in extensions
+}
 
 
 def get_format_from_extension(ext: str) -> FileFormat:

src/polystore/memory.py

@@ -139,6 +139,9 @@ def list_files(
         if self._memory_store[dir_key] is not None:
             raise NotADirectoryError(f"Path is not a directory: {directory}")
 
+        lowercase_extensions = (
+            None if extensions is None else {extension.lower() for extension in extensions}
+        )
         result = []
         dir_prefix = dir_key + "/" if not dir_key.endswith("/") else dir_key
 
@@ -159,7 +162,10 @@ def list_files(
                 filename = Path(rel_path).name
                 # If pattern is None, match all files
                 if pattern is None or fnmatch(filename, pattern):
-                    if not extensions or Path(filename).suffix in extensions:
+                    if (
+                        lowercase_extensions is None
+                        or Path(filename).suffix.lower() in lowercase_extensions
+                    ):
                         # Calculate depth for breadth-first sorting
                         depth = rel_path.count('/')
                         result.append((Path(path), depth))
@@ -651,3 +657,6 @@ def __init__(self, target: str):
 
     def __repr__(self):
         return f"<MemorySymlink → {self.target}>"
+
+
+MemoryBackend = MemoryStorageBackend

src/polystore/napari_stream.py

@@ -20,7 +20,6 @@
 import zmq
 
 from .constants import Backend, TransportMode
-from .streaming_constants import StreamingDataType
 from .streaming import StreamingBackend
 from .roi_converters import NapariROIConverter
 from zmqruntime.transport import get_zmq_transport_url, coerce_transport_mode
@@ -32,12 +31,9 @@ class NapariStreamingBackend(StreamingBackend):
     """Napari streaming backend with automatic registration."""
     _backend_type = Backend.NAPARI_STREAM.value
 
-    # Configure ABC attributes
     VIEWER_TYPE = 'napari'
     SHM_PREFIX = 'napari_'
 
-    # __init__, _get_publisher, save, cleanup now inherited from ABC
-
     def _prepare_shapes_data(self, data: Any, file_path: Union[str, Path]) -> dict:
         """
         Prepare shapes data for transmission.
@@ -57,7 +53,7 @@ def _prepare_shapes_data(self, data: Any, file_path: Union[str, Path]) -> dict:
         }
 
     def _prepare_batch_item(self, data: Any, file_path: Union[str, Path], data_type):
-        if data_type in (StreamingDataType.SHAPES, StreamingDataType.POINTS):
+        if data_type.uses_napari_vector_payload:
             item_data = self._prepare_shapes_data(data, file_path)
             data_type_value = data_type.value
         else:
@@ -88,6 +84,7 @@ def save_batch(self, data_list: List[Any], file_paths: List[Union[str, Path]], *
         microscope_handler = kwargs['microscope_handler']
         source = kwargs.get('source', 'unknown_source')  # Pre-built source value
         plate_path = kwargs.get('plate_path')
+        component_metadata = kwargs.get('component_metadata')
         display_payload_extra = {
             "colormap": display_config.get_colormap_name(),
             "variable_size_handling": display_config.variable_size_handling.value
@@ -103,6 +100,7 @@ def save_batch(self, data_list: List[Any], file_paths: List[Union[str, Path]], *
             display_config,
             self._prepare_batch_item,
             plate_path=plate_path,
+            component_metadata=component_metadata,
             display_payload_extra=display_payload_extra,
         )