diff --git a/.gitconfig b/.gitconfig new file mode 100644 index 000000000..9f4b11401 --- /dev/null +++ b/.gitconfig @@ -0,0 +1,2 @@ +[core] + hooksPath = hooks diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 05d245949..acfc46247 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -46,6 +46,17 @@ on: options: - cpu-only - full-coverage + dependency_source: + description: 'Dependency source for external modules' + required: true + default: 'submodules' + type: choice + options: + - submodules + - pypi + +env: + OPENHCS_CI_DEP_SOURCE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dependency_source || 'submodules' }} jobs: # Group 1: Python version boundary testing (6 jobs) @@ -53,7 +64,7 @@ jobs: # Python 3.14 excluded: no pre-built NumPy wheels yet (builds from source stall on Windows) python-boundary-tests: runs-on: ${{ matrix.os }} - if: github.event_name == 'push' || github.event_name == 'pull_request' + if: github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' strategy: fail-fast: false matrix: @@ -64,12 +75,30 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: ${{ env.OPENHCS_CI_DEP_SOURCE == 'submodules' && 'recursive' || 'false' }} - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install system dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + # ubuntu-latest (23.10+) no longer provides libgl1-mesa-glx; use libgl1. + # PyQt6 requires libEGL.so.1 (EGL) and libGL.so.1 (OpenGL). + sudo apt-get install -y libgl1 + sudo apt-get install -y libegl1 || sudo apt-get install -y libegl1-mesa + sudo apt-get install -y libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 xvfb + + - name: Install local external modules (submodule mode) + if: env.OPENHCS_CI_DEP_SOURCE == 'submodules' + run: | + python -m pip install --upgrade pip + python -c "import subprocess,sys;mods=['external/ObjectState','external/python-introspect','external/metaclass-registry','external/arraybridge','external/pycodify','external/PolyStore','external/pyqt-reactive','external/zmqruntime'];[subprocess.check_call([sys.executable,'-m','pip','install','-e',m]) for m in mods]" + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -145,7 +174,7 @@ jobs: # Test all backend/microscope combinations on all 3 OSes with Python 3.12 backend-microscope-tests: runs-on: ${{ matrix.os }} - if: github.event_name == 'push' || github.event_name == 'pull_request' + if: github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' strategy: fail-fast: false matrix: @@ -156,12 +185,30 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: ${{ env.OPENHCS_CI_DEP_SOURCE == 'submodules' && 'recursive' || 'false' }} - name: Setup Python 3.12 uses: actions/setup-python@v5 with: python-version: "3.12" + - name: Install system dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + # ubuntu-latest (23.10+) no longer provides libgl1-mesa-glx; use libgl1. + # PyQt6 requires libEGL.so.1 (EGL) and libGL.so.1 (OpenGL). + sudo apt-get install -y libgl1 + sudo apt-get install -y libegl1 || sudo apt-get install -y libegl1-mesa + sudo apt-get install -y libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 xvfb + + - name: Install local external modules (submodule mode) + if: env.OPENHCS_CI_DEP_SOURCE == 'submodules' + run: | + python -m pip install --upgrade pip + python -c "import subprocess,sys;mods=['external/ObjectState','external/python-introspect','external/metaclass-registry','external/arraybridge','external/pycodify','external/PolyStore','external/pyqt-reactive','external/zmqruntime'];[subprocess.check_call([sys.executable,'-m','pip','install','-e',m]) for m in mods]" + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -239,7 +286,7 @@ jobs: # See: https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/tag/20240202 omero-tests-linux: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' + if: github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' strategy: fail-fast: false matrix: @@ -253,12 +300,29 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: ${{ env.OPENHCS_CI_DEP_SOURCE == 'submodules' && 'recursive' || 'false' }} - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install system dependencies (Linux) + run: | + sudo apt-get update + # ubuntu-latest (23.10+) no longer provides libgl1-mesa-glx; use libgl1. + # PyQt6 requires libEGL.so.1 (EGL) and libGL.so.1 (OpenGL). + sudo apt-get install -y libgl1 + sudo apt-get install -y libegl1 || sudo apt-get install -y libegl1-mesa + sudo apt-get install -y libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 xvfb + + - name: Install local external modules (submodule mode) + if: env.OPENHCS_CI_DEP_SOURCE == 'submodules' + run: | + python -m pip install --upgrade pip + python -c "import subprocess,sys;mods=['external/ObjectState','external/python-introspect','external/metaclass-registry','external/arraybridge','external/pycodify','external/PolyStore','external/pyqt-reactive','external/zmqruntime'];[subprocess.check_call([sys.executable,'-m','pip','install','-e',m]) for m in mods]" + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -299,11 +363,13 @@ jobs: # Code quality checks (linting, formatting, type checking) code-quality: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' + if: github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: ${{ env.OPENHCS_CI_DEP_SOURCE == 'submodules' && 'recursive' || 'false' }} - name: Setup Python uses: actions/setup-python@v5 @@ -333,7 +399,7 @@ jobs: # Test PyPI-style installation across Python versions and OSes (6 jobs) pypi-installation-test: runs-on: ${{ matrix.os }} - if: github.event_name == 'push' || github.event_name == 'pull_request' + if: github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' strategy: fail-fast: false matrix: @@ -343,6 +409,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: ${{ env.OPENHCS_CI_DEP_SOURCE == 'submodules' && 'recursive' || 'false' }} - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -401,17 +469,27 @@ jobs: # Run integration tests against wheel installation wheel-integration-test: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v') + if: github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: ${{ env.OPENHCS_CI_DEP_SOURCE == 'submodules' && 'recursive' || 'false' }} - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.12" + - name: Install system dependencies (Linux) + run: | + sudo apt-get update + # PyQt6 requires libEGL.so.1 (EGL) and libGL.so.1 (OpenGL). + sudo apt-get install -y libgl1 + sudo apt-get install -y libegl1 || sudo apt-get install -y libegl1-mesa + sudo apt-get install -y libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 xvfb + - name: Build wheel run: | python -m pip install --upgrade pip build @@ -435,4 +513,3 @@ jobs: name: coverage-wheel-integration path: .coverage.wheel retention-days: 1 - diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 333f302f1..560aa9022 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -41,4 +41,4 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | twine check dist/* - twine upload dist/* + twine upload dist/* --skip-existing diff --git a/.gitignore b/.gitignore index 250e29bb1..968cc0905 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,4 @@ win_logs.txt # Chunkhound .chunkhound/ docs/generated_md/ +.grepai/ diff --git a/.gitmodules b/.gitmodules index 44f34db2b..44072e14e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,7 @@ [submodule "external/python-introspect"] path = external/python-introspect url = https://github.com/OpenHCSDev/python-introspect.git - branch = master + branch = main [submodule "external/arraybridge"] path = external/arraybridge url = https://github.com/OpenHCSDev/arraybridge.git diff --git a/CHANGELOG.md b/CHANGELOG.md index ebd8261a2..34884a45e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -#### LLM-Powered Code Generation +#### Auto-Add Output Plate Feature +- **Automatic output plate registration** - New `auto_add_output_plate_to_plate_manager` config option in GlobalPipelineConfig + - When enabled, successfully completed plate runs automatically add the computed output plate to Plate Manager + - Allows immediate visualization of processed results without manual plate addition + - Environment variable: `OPENHCS_AUTO_ADD_OUTPUT_PLATE` + - Accessible via Config Window under Pipeline Settings + +#### Compilation Architecture Improvements +- **Remote compilation via ZMQ server** - Compilation now happens on the ZMQ execution server instead of locally + - New shared `ZMQClientService` for managing ZMQ client connections between compilation and execution + - `CompilationService` and `ZMQExecutionService` now share a single client service instance + - Improved consistency and resource management across compile/run workflows + - Requires ZMQ server to be running for compilation (same as execution) + +#### LLM-Powered Code Generation Improvements - **LLM Assist in Code Editor** - Added LLM-powered code generation directly into the OpenHCS code editor + - **Automatic array backend handling** - Generated functions no longer require manual backend conversions (`cp.asnumpy()`, `cle.pull()`) + - **Context-aware system prompts** - New `get_system_prompt(code_type)` method provides different prompts for pipeline vs function generation + - **Array format clarification** - Documentation now specifies (C, Y, X) a.k.a. (Z, Y, X) format for 3D arrays + - **Memory decorator awareness** - LLM understands `@numpy`, `@cupy`, `@pyclesperanto` decorators and their implications - New `LLMPipelineService` for communicating with local LLM endpoints (Ollama) - New `LLMChatPanel` widget with chat interface for natural language code generation - Integrated LLM panel into `QScintillaCodeEditorDialog` with toggle button diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..641611ea9 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +.PHONY: dev install-hooks help + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +dev: install-hooks ## Set up development environment (installs hooks and dependencies) + pip install -e ".[dev]" + @echo "✅ Development environment ready!" + @echo "Hooks are installed and will auto-update submodules on git operations." + +install-hooks: ## Install git hooks for submodule auto-update + @git config core.hooksPath hooks + @echo "✅ Git hooks installed" + +update-submodules: ## Manually update all submodules to latest + git submodule update --remote + +status: ## Show submodule status + git submodule status diff --git a/README.md b/README.md index 36f82405e..e354b0146 100644 --- a/README.md +++ b/README.md @@ -1,625 +1,396 @@ -# OpenHCS: Open High-Content Screening - - -[![PyPI version](https://img.shields.io/pypi/v/openhcs.svg)](https://pypi.org/project/openhcs/) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -[![Python 3.12+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/) -[![GPU Accelerated](https://img.shields.io/badge/GPU-Accelerated-green.svg)](https://github.com/trissim/openhcs) -[![Documentation Status](https://readthedocs.org/projects/openhcs/badge/?version=latest)](https://openhcs.readthedocs.io/en/latest/?badge=latest) -**A bioimage analysis platform for high-content screening with compile-time validation and bidirectional GUI-code conversion.** +
+  ___                    _   _  ___  _____
+ / _ \ _ __  ___  _ ___ | | | |/ __\/ ___/
+| | | | '_ \/ _ \| '_  \| |_| | |   \___ \
+| |_| | |_||| __/| | | ||  _  | |__  __/ |
+ \___/| .__/\___||_| |_||_| |_|\___/\____/
+      |_|           High-Content Screening
+
-OpenHCS is designed to handle large microscopy datasets (100GB+) with an architecture that emphasizes early error detection and flexible workflows. The platform provides compile-time pipeline validation, live configuration updates across windows, and bidirectional conversion between GUI and code representations. +**Bioimage analysis platform for high-content screening**\ +**Compile-time validation · Bidirectional GUI↔Code · Multi-GPU · LLM pipeline generation · 574+ functions** -## Key Features - -### 1. Compile-Time Pipeline Validation +[![PyPI version](https://img.shields.io/pypi/v/openhcs.svg)](https://pypi.org/project/openhcs/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/) +[![GPU Accelerated](https://img.shields.io/badge/GPU-Accelerated-green.svg)](https://github.com/OpenHCSDev/OpenHCS) +[![Documentation](https://readthedocs.org/projects/openhcs/badge/?version=latest)](https://openhcs.readthedocs.io) -Many bioimage analysis tools validate pipelines at runtime, which can lead to failures after hours of processing. OpenHCS uses a 5-phase compilation system to catch errors before execution starts: + -```python -# Compilation produces immutable execution contexts -for well_id in wells_to_process: - context = self.create_context(well_id) - - # 5-Phase Compilation - fails BEFORE execution starts - PipelineCompiler.initialize_step_plans_for_context(context, pipeline_definition) - PipelineCompiler.declare_zarr_stores_for_context(context, pipeline_definition, self) - PipelineCompiler.plan_materialization_flags_for_context(context, pipeline_definition, self) - PipelineCompiler.validate_memory_contracts_for_context(context, pipeline_definition, self) - PipelineCompiler.assign_gpu_resources_for_context(context) - - context.freeze() # Immutable - prevents state mutation during execution - compiled_contexts[well_id] = context -``` +--- -This approach catches errors at compile time rather than after hours of processing. Immutable frozen contexts help prevent state mutation bugs during execution. +## 🎬 Demo -### 2. Live Cross-Window Configuration Updates +[![Watch OpenHCS demo (5 min)](docs/source/_static/ui.png)](https://openhcs.readthedocs.io/en/latest/_static/openhcs.mp4) -Configuration changes propagate across windows in real-time using lazy resolution with Python's contextvars and MRO-based inheritance: +Watch demo in browser player: https://openhcs.readthedocs.io/en/latest/_static/openhcs.mp4 +Mirror link (GitHub raw): https://raw.githubusercontent.com/OpenHCSDev/OpenHCS/refs/heads/main/docs/source/_static/openhcs.mp4 -- Open 3 windows simultaneously: GlobalPipelineConfig, PipelineConfig, StepConfig -- Edit a value in GlobalPipelineConfig -- Watch placeholders update in real-time in PipelineConfig and StepConfig windows -- Proper inheritance chain: Global → Pipeline → Step with scope isolation per orchestrator -- Save PipelineConfig, and step editors immediately use the new saved values +--- -This uses a class-level registry of active form managers, Qt signals for cross-window updates, and contextvars-based context stacking with MRO-based dual-axis resolution. +OpenHCS processes large microscopy datasets (100GB+) with a **compile-then-execute** architecture. Pipelines are validated across all wells *before* any processing starts, preventing failures after hours of computation. Design pipelines in the GUI, export to Python, edit as code, and re-import — switching seamlessly between visual and programmatic workflows. -### 3. Bidirectional UI-Code Conversion +```mermaid +graph LR + subgraph Microscopes + IX[ImageXpress] + OP[Opera Phenix] + OM[OMERO] + end -Pipelines can be designed in the GUI, exported to Python code, edited as code, and re-imported back to the GUI with full fidelity: + subgraph OpenHCS Platform + PD["Pipeline Designer
(GUI ⇄ Code ⇄ LLM)"] + CO["5-Phase Compiler
(validate)"] + EX["Multi-Process Executor
(1 process/well · multi-GPU)"] + FN["574+ Unified Functions
scikit-image · CuPy · pyclesperanto
PyTorch · JAX · TF · CuCIM · custom"] + PS["PolyStore
(Memory ↔ Disk ↔ ZARR ↔ Stream)"] + end -1. **Design in GUI**: Build pipeline visually with drag-and-drop -2. **Export to Code**: Click "Code" button → get complete executable Python script -3. **Edit in Code**: Bulk modifications, complex parameter tuning, version control -4. **Re-import to GUI**: Save edited code → GUI updates with all changes -5. **Repeat**: Switch between representations seamlessly + subgraph Viewers + NA[Napari] + FJ[Fiji/ImageJ] + end -**Three-Tier Generation Architecture**: + IX --> PD + OP --> PD + OM --> PD + PD --> CO --> EX + EX --> FN --> PS + PS --> NA + PS --> FJ ``` -Function Patterns (Tier 1) - ↓ (encapsulates imports) -Pipeline Steps (Tier 2) - ↓ (encapsulates all pattern imports) -Orchestrator Config (Tier 3) - ↓ (encapsulates all pipeline imports) -Complete Executable Script -``` - -This enables visual tools for rapid prototyping, code editing for complex modifications, and version control for collaboration. - -### 4. Large Dataset Support - -OpenHCS is designed to handle large high-content screening datasets (100GB+): - -- **Virtual File System**: Automatic backend switching between memory, disk, and ZARR storage -- **OME-ZARR Compression**: Configurable algorithms (LZ4, ZLIB, ZSTD, Blosc) with adaptive chunking -- **GPU Resource Management**: Automatic assignment and load balancing across multiple GPUs -- **Parallel Processing**: Scales to arbitrary CPU cores with configurable worker processes - -For example, processing entire 96-well plates with 9 sites per well, 4 channels, and 100+ timepoints (100GB+ per plate). -## Background +--- -OpenHCS evolved from EZStitcher, a microscopy stitching library, into a more general bioimage analysis platform. The architecture addresses some common challenges in scientific software: +## ⚡ Key Capabilities -- Early error detection through compile-time validation -- Flexible workflows that work both in GUI and as code -- Handling datasets that exceed available memory -- Multi-GPU resource management + + + + + + + + + + + + + + + + + +
-### Architecture +### 🛡️ Compile-Time Validation +Pipelines are compiled through **5 phases** before execution — path planning, store declaration, materialization flags, memory contract validation, GPU assignment — then frozen into immutable contexts. Errors surface immediately, not after hours of processing. -The codebase implements several patterns that may be of interest: + -1. **Dual-Axis Configuration Framework**: Combines context hierarchy (global → pipeline → step) with class inheritance (MRO) for configuration resolution. Extracted as standalone library: [hieraconf](https://github.com/trissim/hieraconf) +### 🔄 Bidirectional GUI ↔ Code +Design pipelines visually, export as executable Python, edit in your IDE, re-import to the GUI. Code generation works at **any scope level** — function patterns, individual steps, pipeline configs, full orchestrator scripts — any window holding objects can generate and re-import code. -2. **Lazy Dataclass Factory**: Runtime generation of configuration classes with `__getattribute__` interception for on-demand resolution. +
-3. **Cross-Window Live Updates**: Class-level registry of active form managers with Qt signals for propagating changes across windows. +### 🧠 LLM Pipeline Generation +Describe a pipeline in natural language and get executable code. Built-in chat panel with local Ollama or remote LLM endpoints. Dynamic system prompts built from the actual function registry — the LLM knows every available function and its signature. -4. **5-Phase Pipeline Compiler**: Separates pipeline definition from execution to enable compile-time validation. + -5. **Bidirectional Code Generation**: Three-tier generation system (function patterns → pipeline steps → orchestrator) for round-trip conversion between GUI and code. +### ⚡ Full Multiprocessing & Multi-GPU +**1 process per well** via `ProcessPoolExecutor` with GPU scheduler assigning devices to workers. CUDA spawn-safe. Scales from laptops to multi-GPU workstations — process 100GB+ datasets with OME-ZARR compression (LZ4, ZSTD, Blosc). -**See**: [Architecture Documentation](https://openhcs.readthedocs.io/en/latest/architecture/) for detailed technical analysis. +
-## Flexible Pipeline Platform +### 🔌 Any Python Function +Register **any** Python function by decorating it with `@numpy`, `@cupy`, `@pyclesperanto`, `@torch`, or other memory type decorators. Custom functions get automatic contract validation, UI integration, and appear alongside built-in functions. Persisted to `~/.openhcs/custom_functions/`. -### General-Purpose Bioimage Analysis -OpenHCS provides a flexible platform for creating custom image analysis pipelines. Researchers can combine processing functions to build workflows tailored to their experimental needs. + -### Function Library -The platform automatically discovers and integrates 574+ functions from multiple libraries (pyclesperanto, CuPy, PyTorch, JAX, TensorFlow, scikit-image), providing unified access to image processing, segmentation, and analysis tools. +### 📊 Results Materialization +`@special_outputs` declaratively routes analysis results to **CSV**, **JSON**, **ROI ZIP** (ImageJ-compatible), or **TIFF stacks** via pluggable format writers. ROIs stream to Fiji/ImageJ. Results appear alongside processed images with no manual I/O code. -### Custom Function Integration -Adding custom functions requires following simple signature conventions, allowing the platform to automatically discover and incorporate new processing capabilities. +
-## Supported Microscope Systems +### 🔬 Process-Isolated Napari & Fiji +Stream images to **Napari** and **Fiji/ImageJ** in real-time during pipeline execution. Viewers run in separate processes via ZeroMQ — no Qt threading conflicts. Persistent viewers survive pipeline completion. PolyStore treats viewers as streaming backends. -OpenHCS provides unified interfaces for multiple microscope formats with automatic format detection: + -- **ImageXpress (Molecular Devices)**: Complete support for high-content screening systems including metadata parsing and multi-well organization -- **Opera Phenix (PerkinElmer)**: Automated microscopy platform integration with full metadata support -- **OpenHCS Format**: Optimized internal format for maximum performance and compression -- **Extensible Architecture**: Framework for adding new microscope types without code changes +### 🪟 Live Cross-Window Updates +Edit a value in `GlobalPipelineConfig` — watch it propagate in real-time to `PipelineConfig` and `StepConfig` windows. Dual-axis resolution (context hierarchy × class MRO) with scope isolation per orchestrator. -## Desktop Interface and Workflow +
-### Visual Pipeline Editor -The PyQt6 desktop interface provides drag-and-drop pipeline creation with real-time parameter adjustment and live preview of processing results. +--- -### Bidirectional Code Integration -Pipelines can be exported as executable Python scripts for customization, then re-imported back to the interface. +## 🧩 The OpenHCS Ecosystem -### Real-Time Visualization -Integrated napari viewers provide immediate visualization of processing results, with persistent viewers that survive pipeline completion for examining intermediate results. +OpenHCS is built on **8 purpose-extracted libraries** — each solving a general problem, each independently publishable, all woven into a cohesive platform: -## Installation +```mermaid +graph TD + OH["OpenHCS Platform
(domain wiring + pipelines)"] -OpenHCS is available on PyPI and requires Python 3.11+ with optional GPU acceleration support for CUDA 12.x. + OH --> OS["ObjectState
(config)"] + OH --> AB["ArrayBridge
(arrays)"] + OH --> PS["PolyStore
(I/O + streaming)"] + OH --> ZR["ZMQRuntime
(exec)"] + OH --> QR["PyQT-reactive
(forms)"] -### Quick Start - -```bash -# Desktop GUI (recommended for most users) -pip install openhcs[gui] - -# Then launch the application -openhcs + OS --> PI["python-introspect
(signatures)"] + OH --> MR["metaclass-registry
(plugins)"] + OH --> PC["pycodify
(serialization)"] ``` -### Installation Options +| Library | Role in OpenHCS | What It Does | +|:--------|:----------------|:-------------| +| [**ObjectState**](https://github.com/OpenHCSDev/ObjectState) | Configuration framework | Lazy dataclasses with dual-axis inheritance (context hierarchy × class MRO) and `contextvars`-based resolution | +| [**ArrayBridge**](https://github.com/OpenHCSDev/ArrayBridge) | Memory type conversion | Unified API across NumPy, CuPy, PyTorch, JAX, TensorFlow, pyclesperanto with DLPack zero-copy transfers | +| [**PolyStore**](https://github.com/OpenHCSDev/PolyStore) | Unified I/O & streaming | Pluggable backends for storage (disk, memory, ZARR) *and* streaming (Napari, Fiji) — viewers are just backends. Virtual workspace, atomic writes, format detection, ROI extraction | +| [**ZMQRuntime**](https://github.com/OpenHCSDev/ZMQRuntime) | Distributed execution | ZMQ client-server for remote pipeline execution, progress streaming, and OMERO server-side processing | +| [**PyQT-reactive**](https://github.com/OpenHCSDev/PyQT-reactive) | UI form generation | React-style reactive forms from dataclasses with cross-window sync and flash animations | +| [**pycodify**](https://github.com/OpenHCSDev/pycodify) | Code ↔ object conversion | Python source as serialization format — type-preserving, diffable, editable, with collision handling | +| [**python-introspect**](https://github.com/OpenHCSDev/python-introspect) | Signature analysis | Pure-Python function/dataclass introspection for automatic UI generation and contract analysis | +| [**metaclass-registry**](https://github.com/OpenHCSDev/metaclass-registry) | Plugin discovery | Zero-boilerplate registry system powering microscope handler and storage backend auto-discovery | -```bash -# Headless (servers, CI, programmatic use - no GUI) -pip install openhcs +--- -# Desktop GUI only -pip install openhcs[gui] +## 🔬 Microscope & Function Support -# GUI + Napari viewer -pip install openhcs[gui,napari] + + + + + +
-# GUI + Fiji/ImageJ viewer -pip install openhcs[gui,fiji] +**Microscope Systems** -# GUI + both viewers -pip install openhcs[gui,viz] +| System | Vendor | +|:-------|:-------| +| ImageXpress | Molecular Devices | +| Opera Phenix | PerkinElmer | +| OMERO | Open Microscopy | +| OpenHCS Format | Native | -# Full installation (GUI + viewers + GPU) -pip install openhcs[gui,viz,gpu] +Auto-detected. Extensible via `metaclass-registry`. -# Headless with GPU (server processing) -pip install openhcs[gpu] + -# OMERO integration -pip install openhcs[omero] -``` +**574+ Functions — Automatic Discovery** -### OMERO Integration +| Library | Functions | Acceleration | +|:--------|:---------:|:------------:| +| pyclesperanto | 230+ | OpenCL GPU | +| CuCIM/CuPy | 124+ | CUDA GPU | +| scikit-image | 110+ | CPU | +| PyTorch / JAX / TF | ✓ | CUDA GPU | +| OpenHCS native | ✓ | Mixed | -The OMERO integration requires the `zeroc-ice` dependency, which is not available on PyPI. The custom `setup.py` automatically downloads and installs zeroc-ice when installing with OMERO extras. +Unified contracts, automatic memory conversion via `ArrayBridge`. -#### Automatic Installation (Recommended) +
-```bash -# Install openhcs with OMERO support - zeroc-ice is installed automatically! -pip install 'openhcs[omero]' -``` +**Processing domains**: image preprocessing · segmentation · cell counting · stitching (MIST + Ashlar GPU) · neurite tracing · morphology · measurements -The custom `setup.py` automatically detects your Python version (3.11 or 3.12) and platform (Windows, Linux, macOS), then downloads and installs the appropriate zeroc-ice wheel from Glencoe Software's repository. +--- -**For editable installs (development):** -```bash -pip install -e ".[omero]" -``` - -#### Alternative Installation Methods - -If automatic installation fails, you can use one of these alternatives: - -**Option 1: Standalone Script** -```bash -# Install zeroc-ice using the standalone script -python scripts/install_omero_deps.py - -# Then install openhcs with OMERO support -pip install 'openhcs[omero]' -``` - -**Option 2: Requirements File** -```bash -# Install all OMERO dependencies at once -pip install -r requirements-omero.txt -``` - -This uses `--find-links` to point to Glencoe Software's wheel repository. - -**Option 3: Manual Installation** -1. Visit [Glencoe Software's Ice Binaries](https://www.glencoesoftware.com/blog/2023/12/08/ice-binaries-for-omero.html) -2. Download the appropriate `zeroc_ice-3.7.9-py3-none-any.whl` for your Python version -3. Install the wheel: - ```bash - pip install /path/to/zeroc_ice-3.7.9-py3-none-any.whl - ``` -4. Then install openhcs with OMERO support: - ```bash - pip install 'openhcs[omero]' - ``` +## 🚀 Quick Start -**Note**: OMERO integration is supported on Python 3.11 and 3.12 only. - -**Optional Advanced Features**: ```bash -# GPU-accelerated Viterbi decoding for neurite tracing -pip install git+https://github.com/trissim/torbi.git - -# JAX-based BaSiC illumination correction (optional, numpy/cupy versions included) -pip install basicpy -``` - -### Development Installation - -```bash -# Clone the repository -git clone https://github.com/trissim/openhcs.git -cd openhcs - -# Install with all features for development -pip install -e ".[all]" -``` - -### GPU Requirements - -GPU acceleration requires CUDA 12.x. For CPU-only operation: - -```bash -# Skip GPU dependencies entirely -export OPENHCS_CPU_ONLY=true +# Basic installation with GUI pip install openhcs[gui] -``` - -### Launch Application - -After installing with `[gui]`, launch the desktop interface: - -```bash -# Launch GUI (requires openhcs[gui]) -openhcs - -# Alternative commands -openhcs-gui # Same as 'openhcs' -python -m openhcs.pyqt_gui # Module invocation -# With debug logging -openhcs --log-level DEBUG - -# Show help -openhcs --help -``` - -**Note**: The `openhcs` command requires GUI dependencies. If you installed headless (`pip install openhcs`), you'll get a helpful error message telling you to install `openhcs[gui]`. - -## Basic Usage - -### Getting Started +# Add Napari viewer +pip install openhcs[gui,napari] -OpenHCS provides a desktop interface for interactive pipeline creation and execution. The application guides users through microscopy data selection, pipeline configuration, and analysis execution. +# Add Fiji/ImageJ viewer +pip install openhcs[gui,fiji] -```python -from openhcs.core.orchestrator.pipeline_orchestrator import PipelineOrchestrator -from openhcs.core.config import GlobalPipelineConfig +# Add both viewers +pip install openhcs[gui,viz] -# Initialize OpenHCS -orchestrator = PipelineOrchestrator( - input_dir="path/to/microscopy/data", - global_config=GlobalPipelineConfig(num_workers=4) -) +# Add GPU acceleration (CUDA 12.x required) +pip install openhcs[gui,gpu] -# Initialize the orchestrator -orchestrator.initialize() +# Full installation (GUI + viewers + GPU) +pip install openhcs[gui,viz,gpu] -# Run complete analysis pipeline (requires pipeline definition) -# Use the desktop interface to create pipelines interactively +# Launch the application +openhcs ``` -### Pipeline Definition - -OpenHCS pipelines consist of FunctionStep objects that define processing operations. Each step specifies the function to execute, parameters, and data organization strategy: - ```python +# Or use programmatically — real pipeline from a neuroscience experiment +from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.steps.function_step import FunctionStep +from openhcs.constants.constants import VariableComponents from openhcs.processing.backends.processors.cupy_processor import ( stack_percentile_normalize, tophat, create_composite ) -from openhcs.processing.backends.analysis.cell_counting_cupy import count_cells_single_channel from openhcs.processing.backends.pos_gen.ashlar_main_gpu import ashlar_compute_tile_positions_gpu from openhcs.processing.backends.assemblers.assemble_stack_cupy import assemble_stack_cupy -from openhcs.constants.constants import VariableComponents +from openhcs.processing.backends.analysis.cell_counting_cpu import count_cells_single_channel -# Define processing pipeline steps = [ - # Image preprocessing - FunctionStep( - func=[stack_percentile_normalize], - name="normalize", - variable_components=[VariableComponents.SITE] - ), - FunctionStep( - func=[(tophat, {'selem_radius': 25})], - name="enhance", - variable_components=[VariableComponents.SITE] - ), - - # Position generation for stitching - FunctionStep( - func=[ashlar_compute_tile_positions_gpu], - name="positions", - variable_components=[VariableComponents.SITE] - ), - - # Image assembly using calculated positions - FunctionStep( - func=[assemble_stack_cupy], - name="assemble", - variable_components=[VariableComponents.SITE] - ), - - # Cell analysis - FunctionStep( - func=[count_cells_single_channel], - name="count_cells", - variable_components=[VariableComponents.SITE] - ) -] - -# Complete working examples available in openhcs/debug/example_export.py -``` - -## Processing Functions - -OpenHCS provides access to over 574 image processing functions through automatic discovery from multiple libraries: - -### Image Processing -The platform includes comprehensive image processing capabilities: normalization and denoising for preprocessing, Gaussian and median filtering for noise reduction, morphological operations including opening and closing, and projection operations for dimensionality reduction. - -### Cell Analysis -Cell analysis functions support detection through blob detection algorithms (LOG, DOG, DOH), watershed segmentation, and threshold-based methods. GPU-accelerated watershed and region growing provide efficient segmentation. Measurement functions extract intensity, morphology, and texture features from segmented regions. - -### Stitching Algorithms -OpenHCS implements GPU-accelerated versions of established stitching algorithms. MIST provides phase correlation with robust optimization for tile position calculation. Ashlar offers edge-based alignment with GPU acceleration. Assembly functions perform subpixel positioning and blending for final image reconstruction. - -### Neurite Analysis -Specialized neurite analysis includes GPU-accelerated morphological thinning for skeletonization, SKAN-based neurite tracing with HMM models, and quantification of length, branching, and connectivity metrics. - -## Documentation - -Comprehensive documentation covers all aspects of OpenHCS architecture and usage: - -- **[Read the Docs](https://openhcs.readthedocs.io/)** - Complete API documentation, tutorials, and guides -- **[Coverage Reports](https://trissim.github.io/openhcs/coverage/)** - Test coverage analysis -- **[API Reference](https://openhcs.readthedocs.io/en/latest/api/)** - Detailed function and class documentation -- **[User Guide](https://openhcs.readthedocs.io/en/latest/user_guide/)** - Step-by-step tutorials and examples - -### Key Documentation Sections -- **Architecture**: [Pipeline System](https://openhcs.readthedocs.io/en/latest/architecture/pipeline-compilation-system.html) | [GPU Processing](https://openhcs.readthedocs.io/en/latest/architecture/gpu-resource-management.html) | [VFS](https://openhcs.readthedocs.io/en/latest/architecture/vfs-system.html) -- **Getting Started**: [Installation](https://openhcs.readthedocs.io/en/latest/getting_started/installation.html) | [First Pipeline](https://openhcs.readthedocs.io/en/latest/getting_started/first_pipeline.html) -- **Advanced Topics**: [GPU Optimization](https://openhcs.readthedocs.io/en/latest/guides/gpu_optimization.html) | [Large Datasets](https://openhcs.readthedocs.io/en/latest/guides/large_datasets.html) - -## Technical Architecture Deep Dive + FunctionStep(func=[ + (stack_percentile_normalize, {'low_percentile': 1.0, 'high_percentile': 99.0}), + (tophat, {'selem_radius': 50, 'downsample_factor': 4}) + ], name="preprocess", variable_components=[VariableComponents.SITE]), -OpenHCS demonstrates several architectural patterns applicable beyond microscopy. The codebase is worth studying for its novel approaches to common software engineering challenges. + FunctionStep(func=[create_composite], + name="composite", variable_components=[VariableComponents.CHANNEL]), -### 5-Phase Pipeline Compilation System + FunctionStep(func=[ashlar_compute_tile_positions_gpu], + name="find_positions", variable_components=[VariableComponents.SITE]), -**Problem**: Traditional scientific software fails at runtime after hours of processing. + FunctionStep(func=[(assemble_stack_cupy, {'blend_method': 'fixed'})], + name="assemble", variable_components=[VariableComponents.SITE], + force_disk_output=True), -**Solution**: Declarative compilation architecture that validates entire processing chains before execution. + FunctionStep(func=[count_cells_single_channel], + name="count_cells", variable_components=[VariableComponents.SITE]), +] -**Implementation**: -```python -# Compilation produces immutable execution contexts -for well_id in wells_to_process: - context = self.create_context(well_id) - - # 5-Phase Compilation - fails BEFORE execution starts - PipelineCompiler.initialize_step_plans_for_context(context, pipeline_definition) - PipelineCompiler.declare_zarr_stores_for_context(context, pipeline_definition, self) - PipelineCompiler.plan_materialization_flags_for_context(context, pipeline_definition, self) - PipelineCompiler.validate_memory_contracts_for_context(context, pipeline_definition, self) - PipelineCompiler.assign_gpu_resources_for_context(context) - - context.freeze() # Immutable - prevents state mutation during execution - compiled_contexts[well_id] = context +orchestrator = PipelineOrchestrator("path/to/plate") +orchestrator.initialize() +compiled = orchestrator.compile_pipelines(steps) # Validates everything first +orchestrator.execute_compiled_plate(steps, compiled, max_workers=5) ``` -**Key Innovations**: -- Immutable frozen contexts prevent state mutation bugs -- Compile-time validation catches errors before execution -- Separation of compilation and execution phases -- GPU resource assignment at compile time, not runtime - -**See**: [Pipeline Compilation System](https://openhcs.readthedocs.io/en/latest/architecture/pipeline-compilation-system.html) +
+📦 All installation options -### Dual-Axis Configuration Framework - -**Problem**: Configuration systems typically support either hierarchy (global → local) OR inheritance (class-based), not both. - -**Solution**: Dual-axis resolution combining context hierarchy with class inheritance (MRO). - -**Implementation**: -```python -# Lazy dataclass with __getattribute__ interception -class LazyPipelineConfig(PipelineConfig): - def __getattribute__(self, name): - # Stage 1: Check instance attributes (user overrides) - # Stage 2: Check context stack (global → pipeline → step) - # Stage 3: Walk MRO for class-level defaults - # Stage 4: Return None if no value found +```bash +pip install openhcs # Headless (servers, CI) +pip install openhcs[gui] # Desktop GUI +pip install openhcs[gui,napari] # GUI + Napari viewer +pip install openhcs[gui,viz] # GUI + Napari + Fiji +pip install openhcs[gui,viz,gpu] # Full installation +pip install openhcs[gpu] # Headless + GPU +pip install openhcs[omero] # OMERO integration +pip install -e ".[all,dev]" # Development (all features) ``` -**Key Innovations**: -- Preserves None vs concrete value distinction for proper inheritance -- Contextvars-based context stacking for thread-safe resolution -- MRO-based dual-axis resolution (context + class hierarchy) -- Field-level inheritance (different fields can inherit from different sources) - -**Extracted as standalone library**: [hieraconf](https://github.com/trissim/hieraconf) +GPU requires CUDA 12.x. For CPU-only: `OPENHCS_CPU_ONLY=true pip install openhcs[gui]` -**See**: [Configuration Framework](https://openhcs.readthedocs.io/en/latest/architecture/configuration_framework.html) +
-### Cross-Window Live Updates +
+🗄️ OMERO integration -**Problem**: Most GUI applications treat each window as isolated. Configuration changes require close-reopen cycles. +OMERO requires `zeroc-ice` (not on PyPI). The custom `setup.py` installs it automatically: -**Solution**: Class-level registry of active form managers with Qt signals for cross-window updates. - -**Implementation**: -```python -# Class-level registry tracks all active form managers -_active_form_managers = [] - -# When a value changes in one window -def _emit_cross_window_change(self, param_name: str, value: object): - field_path = f"{self.field_id}.{param_name}" - self.context_value_changed.emit(field_path, value, - self.object_instance, self.context_obj) - -# Other windows receive the signal and refresh -def _on_cross_window_context_changed(self, field_path, new_value, - editing_object, context_object): - if not self._is_affected_by_context_change(editing_object, context_object): - return - self._schedule_cross_window_refresh() # Debounced refresh +```bash +pip install 'openhcs[omero]' # Auto-installs zeroc-ice ``` -**Key Innovations**: -- Live context collection from other open windows -- Scope isolation (per-orchestrator) prevents cross-contamination -- Debounced updates prevent excessive refreshes -- Cascading placeholder refreshes (Global → Pipeline → Step) - -**See**: [Parameter Form Lifecycle](https://openhcs.readthedocs.io/en/latest/architecture/parameter_form_lifecycle.html) - -### Bidirectional UI-Code Interconversion - -**Problem**: GUI tools can export to code but can't re-import. You're forced to choose between GUI or code. +If that fails, alternatives: +```bash +python scripts/install_omero_deps.py # Standalone script +pip install -r requirements-omero.txt # Requirements file +``` -**Solution**: Three-tier generation system with perfect round-trip integrity. +Supported on Python 3.11 and 3.12. See [Glencoe Software](https://www.glencoesoftware.com/blog/2023/12/08/ice-binaries-for-omero.html) for manual installation. -**Implementation**: -```python -# Tier 1: Function Pattern Generation -pattern = gaussian_filter(sigma=2.0, preserve_dtype=True) - -# Tier 2: Pipeline Step Generation (encapsulates Tier 1 imports) -step_1 = FunctionStep( - func=(gaussian_filter, {'sigma': 2.0, 'preserve_dtype': True}), - name="gaussian_filter", - variable_components=[VariableComponents.PLATE] -) +
-# Tier 3: Orchestrator Config (encapsulates Tier 1 + 2 imports) -global_config = GlobalPipelineConfig(num_workers=16) -pipeline_data = {plate_path: [step_1, step_2, ...]} -``` +--- -**Key Innovations**: -- Upward import encapsulation (each tier includes all lower tier imports) -- AST-based code parsing for re-import -- Lazy dataclass constructor patching preserves None vs concrete distinction -- Complete executability (generated code runs without additional imports) +## 📖 Documentation -**See**: [Code/UI Interconversion](https://openhcs.readthedocs.io/en/latest/architecture/code_ui_interconversion.html) +| | | +|:---|:---| +| 📘 **[Read the Docs](https://openhcs.readthedocs.io/)** | Full API docs, tutorials, guides | +| 📊 **[Coverage Reports](https://trissim.github.io/openhcs/coverage/)** | Test coverage analysis | +| 🏗️ **[Architecture](https://openhcs.readthedocs.io/en/latest/architecture/)** | Pipeline system · GPU management · VFS · Config framework | +| 🎓 **[Getting Started](https://openhcs.readthedocs.io/en/latest/getting_started/)** | Installation · First pipeline | -### Additional Architectural Patterns +--- -**Process-Isolated Real-Time Visualization**: Napari integration via ZeroMQ eliminates Qt threading conflicts. Persistent viewers survive pipeline completion. +## ⚙️ Architecture Highlights -**Automatic Function Discovery**: 574+ functions from multiple GPU libraries with contract analysis and type-safe integration. +
+5-Phase Pipeline Compilation — catch errors before execution starts -**Virtual File System**: Automatic backend switching (memory, disk, ZARR) for 100GB+ datasets with adaptive chunking. +``` +Define → Compile → Freeze → Execute + ├─ 1. Path planning + ├─ 2. ZARR store declaration + ├─ 3. Materialization flags + ├─ 4. Memory contract validation + └─ 5. GPU assignment + → context.freeze() # immutable +``` -**Strict Memory Type Management**: Compile-time validation of memory type compatibility with automatic conversion between array types. +Pipelines are compiled for **every well** before any processing begins. Frozen contexts prevent state mutation during execution. [Read more →](https://openhcs.readthedocs.io/en/latest/architecture/pipeline-compilation-system.html) -**Evolution-Proof UI Generation**: Type-based form generation from Python annotations. Adapts automatically when signatures change. +
-**See**: [Complete Architecture Documentation](https://openhcs.readthedocs.io/en/latest/architecture/) +
+Dual-Axis Configuration — context hierarchy × class MRO +Resolution walks two axes simultaneously: the **context stack** (Global → Pipeline → Step) and the **class MRO** (inheritance chain). Built on `contextvars` for thread-safe, scope-isolated resolution. Preserves `None` vs concrete value distinction for proper field-level inheritance. Powered by `ObjectState`. [Read more →](https://openhcs.readthedocs.io/en/latest/architecture/configuration_framework.html) +
-## Example Workflows +
+Bidirectional GUI ↔ Code — code generation at any scope level -Complete analysis workflows demonstrate OpenHCS capabilities: +Any window holding `ObjectState` objects can generate and re-import executable Python: -```bash -# View complete production examples -git clone https://github.com/trissim/openhcs.git -cat openhcs/examples/example_export.py +``` +Function patterns · Individual steps · Pipeline configs · Full orchestrator scripts + ↕ generate / AST-parse back ↕ ``` -Example workflows include preprocessing, stitching, and analysis steps with GPU acceleration, large dataset handling through ZARR compression, parallel processing with resource monitoring, and comprehensive configuration management. - -## Who Should Use OpenHCS? - -### For Biologists and Microscopists +Each scope encapsulates all lower-scope imports. Generated code is fully executable without additional setup. Edit in your IDE or external editor, save, and the GUI re-imports via AST parsing. Powered by `pycodify` + `python-introspect`. [Read more →](https://openhcs.readthedocs.io/en/latest/architecture/code_ui_interconversion.html) -**Use OpenHCS if you**: -- Process high-content screening data (96-well plates, multi-site, multi-channel) -- Need to analyze 100GB+ datasets that break CellProfiler or ImageJ -- Want compile-time validation to catch errors before hours of processing -- Need GPU acceleration for faster analysis -- Want to switch between GUI and code without losing work +
-**Don't use OpenHCS if you**: -- Have simple analysis needs (single images, basic measurements) - use ImageJ/Fiji -- Need established community plugins - use CellProfiler -- Don't have Python 3.11+ or can't install dependencies +
+Cross-Window Live Updates — class-level registry + Qt signals -### For Software Engineers and Computer Scientists +A class-level registry tracks all active form managers. When a value changes in any config window, Qt signals propagate the change to every affected window with debounced, scope-isolated refreshes. Global → Pipeline → Step cascading with per-orchestrator isolation. Powered by `PyQT-reactive`. [Read more →](https://openhcs.readthedocs.io/en/latest/architecture/parameter_form_lifecycle.html) -**Study OpenHCS if you're interested in**: -- Novel configuration frameworks (dual-axis resolution, lazy dataclasses) -- Compile-time validation for scientific pipelines -- Cross-window live updates in GUI applications -- Bidirectional UI-code conversion with round-trip integrity -- Metaprogramming patterns (lazy dataclass factory, MRO-based resolution) +
-**The codebase demonstrates**: -- Contextvars-based context stacking for thread-safe resolution -- Immutable frozen contexts preventing state mutation -- Class-level registries for cross-window communication -- AST-based code generation and parsing -- Type-based UI generation from Python annotations +
+More patterns — PolyStore streaming, function discovery, memory types -**Extracted libraries**: -- [hieraconf](https://github.com/trissim/hieraconf) - Hierarchical configuration framework +- **PolyStore Unified I/O**: Storage backends (disk, memory, ZARR) and streaming backends (Napari, Fiji) behind one API — viewers are just backends. Virtual workspace path translation, atomic writes, ROI extraction. +- **Automatic Function Discovery**: 574+ functions with contract analysis and type-safe integration via `python-introspect` + `metaclass-registry` +- **Memory Type Management**: Compile-time validation of array type compatibility with zero-copy conversion via `ArrayBridge` +- **Custom Function Registration**: Any Python function decorated with `@numpy`, `@cupy`, `@pyclesperanto`, etc. is auto-integrated with contracts, UI forms, and the function registry +- **Evolution-Proof UI**: Type-based form generation from Python annotations — adapts automatically when signatures change -**Potential research contributions**: -- Configuration framework patterns (publishable in JOSS or PL conferences) -- Compile-time validation for scientific workflows -- Cross-window live updates architecture +[Full architecture docs →](https://openhcs.readthedocs.io/en/latest/architecture/) -## Contributing +
-OpenHCS welcomes contributions from the scientific computing community. The platform is actively developed for neuroscience research applications. +--- -### Development Setup +## 🤝 Contributing ```bash -# Clone the repository -git clone https://github.com/trissim/openhcs.git -cd openhcs - -# Install in development mode with all features +git clone https://github.com/OpenHCSDev/OpenHCS.git && cd OpenHCS pip install -e ".[all,dev]" - -# Run tests pytest tests/ - -# Run OMERO integration tests (requires Docker) -# See OMERO_TESTING_GUIDE.md for setup instructions -cd openhcs/omero && docker-compose up -d && ./wait_for_omero.sh && cd ../.. -pytest tests/integration/test_main.py --it-microscopes=OMERO --it-backends=disk -v ``` -### Contribution Areas -- **Microscope Formats**: Add support for additional imaging systems -- **Processing Functions**: Contribute specialized analysis algorithms -- **GPU Backends**: Extend support for new GPU computing libraries -- **Documentation**: Improve guides and examples +**Contribution areas**: microscope formats · processing functions · GPU backends · documentation + +--- -## License +## 📄 License -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. +MIT — see [LICENSE](LICENSE). -## Acknowledgments +## 🙏 Acknowledgments -OpenHCS builds upon EZStitcher and incorporates algorithms and concepts from established image analysis libraries including Ashlar for image stitching algorithms, MIST for phase correlation methods, pyclesperanto for GPU-accelerated image processing, and scikit-image for comprehensive image analysis tools. +OpenHCS evolved from [EZStitcher](https://github.com/OpenHCSDev/ezstitcher) and builds on [Ashlar](https://github.com/labsyspharm/ashlar) (stitching), [MIST](https://github.com/usnistgov/MIST) (phase correlation), [pyclesperanto](https://github.com/clEsperanto/pyclesperanto_prototype) (GPU image processing), and [scikit-image](https://scikit-image.org/) (image analysis). diff --git a/docs/plans/zmq-progress-reporting-plan.md b/docs/plans/zmq-progress-reporting-plan.md new file mode 100644 index 000000000..456aeba85 --- /dev/null +++ b/docs/plans/zmq-progress-reporting-plan.md @@ -0,0 +1,147 @@ +# ZMQ Progress Reporting + Pending Status Plan + +## Goals +- Emit detailed per-well/per-step progress from FunctionStep execution over ZMQ. +- Provide percent + phase context (step index, total steps, pattern-group progress). +- Surface “pending” status during plate init and compilation in the Plate Manager. +- Remove legacy progress schema and update all listeners in lockstep (no backward compatibility). + +## Non-Goals +- Textual TUI updates (deprecated). +- Backward compatibility with legacy `progress` payloads. + +## Current Behavior (Summary) +- ZMQ progress is emitted only via zmqruntime’s `progress_queue`, but nothing in OpenHCS feeds it during execution. +- Orchestrator has a `progress_callback`, but it is only used in the single-process path. +- Multiprocessing path uses `_execute_single_axis_static` and never reports progress. +- Plate Manager only updates per-plate label on completion; no pending state for init/compile. + +## New Progress Schema (Strict Migration) +Define a single required schema for all progress messages (no legacy fallback): + +Required fields: +- `type`: `progress` +- `execution_id` +- `plate_id` +- `axis_id` +- `step_name` +- `step_index` +- `total_steps` +- `phase` (enum string: `axis_started`, `step_started`, `pattern_group`, `step_completed`, `axis_completed`, `compile`, `init`) +- `status` (enum string: `started`, `running`, `completed`, `error`) +- `completed` (int) +- `total` (int) +- `percent` (float 0–100) +- `timestamp` + +Optional fields: +- `pattern` (short pattern repr) +- `component` (group_by component value) +- `message` (human-readable) +- `error` / `traceback` + +Example: +```json +{ + "type": "progress", + "execution_id": "...", + "plate_id": "/data/plate_01", + "axis_id": "B06", + "step_name": "SegmentNuclei", + "step_index": 2, + "total_steps": 7, + "phase": "pattern_group", + "status": "running", + "completed": 14, + "total": 50, + "percent": 28.0, + "timestamp": 1700000000.0 +} +``` + +## Implementation Plan + +### 1) zmqruntime: Formalize Progress Schema +Files: +- `external/zmqruntime/src/zmqruntime/messages.py` +- `external/zmqruntime/src/zmqruntime/execution/server.py` +- `external/zmqruntime/src/zmqruntime/execution/client.py` + +Changes: +- Replace `ProgressUpdate` dataclass to reflect the new schema. +- Add new constants to `MessageFields` for the additional fields. +- Update `ExecutionServer.send_progress_update()` to accept a full progress dict or `ProgressUpdate` and enqueue it unchanged. +- Update `ExecutionClient._progress_listener_loop()` to expect the new schema (fail loudly on missing required fields). + +### 2) OpenHCS ZMQ Server: Progress Bridge +Files: +- `openhcs/runtime/zmq_execution_server.py` + +Changes: +- Add a helper on `OpenHCSExecutionServer` to emit progress events using the new schema and include `execution_id`/`plate_id`. +- Create a multiprocessing-safe progress queue for worker processes and forward those events into the server’s `progress_queue`. +- Wire that progress queue into the orchestrator execution path so all worker-side progress reaches ZMQ. + +### 3) Orchestrator: Progress in Both Threading + Multiprocessing +Files: +- `openhcs/core/orchestrator/orchestrator.py` + +Changes: +- Threading path: keep using `progress_callback`, but switch to the new schema payload. +- Multiprocessing path: + - Add a module-level worker progress reporter that pushes events into a multiprocessing queue. + - Pass this queue to the ProcessPool initializer (new init arg) and set a global reporter in workers. + - Emit axis/step start and completion events from `_execute_axis_with_sequential_combinations()` or `_execute_single_axis_static()` using the global reporter. + +### 4) FunctionStep: Fine-Grained Progress Events +Files: +- `openhcs/core/steps/function_step.py` + +Changes: +- Calculate total pattern groups for the current step/well after `prepare_patterns_and_functions()`. +- Emit a `pattern_group` progress update after each processed group. +- Include `completed`, `total`, and `percent` for step-level progress. +- Emit `step_started`/`step_completed` (or `phase` values) for extra clarity. + +### 5) PyQt: Display Rich Progress +Files: +- `openhcs/pyqt_gui/widgets/shared/services/zmq_execution_service.py` +- `openhcs/pyqt_gui/widgets/plate_manager.py` + +Changes: +- Update `_on_progress()` to parse the new schema. +- Maintain per-plate progress state (last step name, percent, phase) for list display. +- Update the status bar message format to include `axis_id`, `step_name`, and percent. + +### 6) Pending Status During Init + Compile +Files: +- `openhcs/pyqt_gui/widgets/plate_manager.py` +- `openhcs/pyqt_gui/widgets/shared/services/compilation_service.py` + +Changes: +- Add per-plate pending flags in Plate Manager (e.g., `init_pending`, `compile_pending`). +- Set `init_pending` when `action_init_plate` begins per plate; clear on success/failure. +- Set `compile_pending` when compile starts in `CompilationService` (before submit); clear on success/failure. +- Update `_format_plate_item_with_preview_text()` to show `⏳ Init` / `⏳ Compile` when pending overrides are active. + +## Update Sites Checklist (All Callers) +- Progress schema source of truth: `external/zmqruntime/src/zmqruntime/messages.py` +- Server-side enqueue: `external/zmqruntime/src/zmqruntime/execution/server.py` +- Client-side listener: `external/zmqruntime/src/zmqruntime/execution/client.py` +- OpenHCS ZMQ server execution flow: `openhcs/runtime/zmq_execution_server.py` +- Orchestrator (threading + multiprocessing): `openhcs/core/orchestrator/orchestrator.py` +- Function-step runtime: `openhcs/core/steps/function_step.py` +- PyQt ZMQ progress UI: `openhcs/pyqt_gui/widgets/shared/services/zmq_execution_service.py` +- Plate Manager pending states: `openhcs/pyqt_gui/widgets/plate_manager.py` +- Compilation service pending hooks: `openhcs/pyqt_gui/widgets/shared/services/compilation_service.py` + +## Migration Strategy (Strict) +- Replace legacy `progress` payloads immediately; no fallback parsing. +- Update all internal listeners in the same change set. +- Add explicit validation in the progress listener to surface schema mismatches quickly. + +## Validation Steps +- Run a small plate with a short pipeline and confirm progress updates show step name + percent. +- Validate pending label transitions: Created -> Init Pending -> Ready, and Ready -> Compile Pending -> Compiled. +- Verify that progress continues to flow in multiprocessing mode. +- Confirm no legacy progress payloads are emitted (grep logs for missing fields). diff --git a/docs/source/_static/openhcs.mp4 b/docs/source/_static/openhcs.mp4 new file mode 100644 index 000000000..e6b8f518f Binary files /dev/null and b/docs/source/_static/openhcs.mp4 differ diff --git a/docs/source/_static/ui.png b/docs/source/_static/ui.png new file mode 100644 index 000000000..e339a4254 Binary files /dev/null and b/docs/source/_static/ui.png differ diff --git a/docs/source/architecture/abstract_manager_widget.rst b/docs/source/architecture/abstract_manager_widget.rst index 6247db330..22b5a2786 100644 --- a/docs/source/architecture/abstract_manager_widget.rst +++ b/docs/source/architecture/abstract_manager_widget.rst @@ -156,7 +156,6 @@ See Also - :doc:`widget_protocol_system` - ABC contracts for widget operations - :doc:`ui_services_architecture` - Service layer for ParameterFormManager - :doc:`gui_performance_patterns` - Cross-window preview system -- :doc:`compilation_service` - Compilation service extracted from PlateManager -- :doc:`zmq_execution_service_extracted` - ZMQ execution service extracted from PlateManager +- :doc:`batch_workflow_service` - Unified compile/execute orchestration service +- :doc:`zmq_server_browser_system` - ZMQ browser abstraction and OpenHCS adapter - :doc:`parametric_widget_creation` - Widget creation configuration - diff --git a/docs/source/architecture/batch_workflow_service.rst b/docs/source/architecture/batch_workflow_service.rst new file mode 100644 index 000000000..7ed3f9b6d --- /dev/null +++ b/docs/source/architecture/batch_workflow_service.rst @@ -0,0 +1,84 @@ +Batch Workflow Service +====================== + +Module +------ + +``openhcs.pyqt_gui.widgets.shared.services.batch_workflow_service`` + +Purpose +------- + +``BatchWorkflowService`` is the single orchestration boundary for: + +- compile-only batches +- execute batches (compile-all first, execute-all after) +- execution completion polling +- progress projection refresh + server-status emission + +Core Types +---------- + +- ``CompileJob``: immutable compile unit +- ``BatchWorkflowService``: end-to-end compile/execute owner + +Dependency Boundary +------------------- + +OpenHCS workflow policy is implemented in ``BatchWorkflowService``. Runtime and +UI polling primitives are consumed as dependencies: + +- ``BatchSubmitWaitEngine`` (from ``zmqruntime.execution``) +- ``ExecutionStatusPoller`` (from ``zmqruntime.execution``) +- ``IntervalSnapshotPoller`` (from ``pyqt_reactive.services``) + +OpenHCS docs define orchestration policy and host contracts only; generic +engine internals are documented in their owning repositories. + +Flow: Compile Only +------------------ + +1. Reset progress views for new batch. +2. Build ``CompileJob`` entries from selected plates. +3. Submit all jobs then wait all jobs via ``BatchSubmitWaitEngine``. +4. Write compiled data into ``host.plate_compiled_data``. +5. Emit orchestrator state transitions and progress updates. + +Flow: Run Batch +--------------- + +1. Reset progress and execution state. +2. Mark all selected plates queued immediately. +3. Compile all selected plates before any execution submission. +4. Submit execution for each plate using ``compile_artifact_id``. +5. Start per-execution completion pollers. +6. Converge to completed/failed/cancelled through one callback path. + +Host Contract (Expected Attributes/Callbacks) +--------------------------------------------- + +``BatchWorkflowService`` expects host-owned UI/state callbacks including: + +- ``emit_status``, ``emit_error``, ``emit_progress_*`` +- ``update_item_list``, ``update_button_states`` +- ``emit_orchestrator_state``, ``emit_execution_complete`` +- ``on_plate_completed``, ``on_all_plates_completed`` +- data stores like ``plate_compiled_data``, ``plate_execution_states``, + ``plate_execution_ids``, and ``_progress_tracker`` + +This is an explicit nominal integration in OpenHCS (not runtime duck-typing). + +Related Modules +--------------- + +- ``openhcs.pyqt_gui.widgets.shared.services.execution_server_status_presenter`` +- ``openhcs.pyqt_gui.widgets.shared.services.progress_batch_reset`` +- ``openhcs.pyqt_gui.widgets.shared.services.plate_config_resolver`` +- ``openhcs.pyqt_gui.widgets.shared.services.zmq_client_service`` + +See Also +-------- + +- :doc:`plate_manager_services` +- :doc:`progress_runtime_projection_system` +- :doc:`zmq_server_browser_system` diff --git a/docs/source/architecture/compilation_service.rst b/docs/source/architecture/compilation_service.rst index f911b9e72..cb93ef2b4 100644 --- a/docs/source/architecture/compilation_service.rst +++ b/docs/source/architecture/compilation_service.rst @@ -1,173 +1,24 @@ -Compilation Service -=================== +Compilation Service (Legacy) +============================ -**Pipeline compilation service extracted from PlateManagerWidget.** +Status +------ -*Module: openhcs.pyqt_gui.widgets.shared.services.compilation_service* +``CompilationService`` has been removed from OpenHCS. -Why Extract Compilation Logic? ------------------------------- +Reason +------ -The original ``PlateManagerWidget`` was over 2,500 lines, mixing UI concerns with -business logic. Compilation—the process of turning a pipeline definition into an -executable orchestrator—is inherently complex, involving: +Compilation and execution orchestration now live in one service boundary to +enforce a single workflow invariant and eliminate duplicated state logic. -- Creating and caching orchestrator instances -- Setting up the context system for worker threads -- Validating pipeline step configurations -- Expanding variable components into iteration sets -- Progress tracking for long-running compilations +Replacement +----------- -None of this requires UI knowledge. By extracting it into ``CompilationService``, -we achieve: +Use :doc:`batch_workflow_service` for all compile and run orchestration. -1. **Testability** - The service can be unit tested without Qt -2. **Reusability** - Other UI components can use the same compilation logic -3. **Maintainability** - UI and business logic evolve independently -4. **Clarity** - Each class has a single, well-defined responsibility - -The Protocol Pattern --------------------- - -The key architectural insight is using a Protocol to define the interface between -service and host. The service doesn't care *what* the host is—it only cares that -the host provides certain attributes and callbacks. This is dependency inversion: -the service depends on an abstraction, not a concrete widget class. - -The Protocol is ``@runtime_checkable``, meaning ``isinstance(obj, CompilationHost)`` -works. This enables fail-loud validation when the service is created: - -.. code-block:: python - - from typing import Protocol, runtime_checkable - - @runtime_checkable - class CompilationHost(Protocol): - """Protocol for widgets that host the compilation service.""" - - # State attributes the service needs - global_config: Any - orchestrators: Dict[str, PipelineOrchestrator] - plate_configs: Dict[str, Dict] - plate_compiled_data: Dict[str, Any] - - # Progress/status callbacks - def emit_progress_started(self, count: int) -> None: ... - def emit_progress_updated(self, value: int) -> None: ... - def emit_progress_finished(self) -> None: ... - def emit_orchestrator_state(self, plate_path: str, state: str) -> None: ... - def emit_compilation_error(self, plate_name: str, error: str) -> None: ... - def emit_status(self, msg: str) -> None: ... - def get_pipeline_definition(self, plate_path: str) -> List: ... - def update_button_states(self) -> None: ... - -Using the Service ------------------ - -Creating the service is straightforward—pass a host that implements the Protocol. -The service stores a reference to the host and calls its methods during compilation: - -.. code-block:: python - - from openhcs.pyqt_gui.widgets.shared.services.compilation_service import ( - CompilationService, CompilationHost - ) - - class MyWidget(QWidget, CompilationHost): - def __init__(self): - super().__init__() - self.compilation_service = CompilationService(host=self) - - async def compile_selected(self): - selected = [{'path': '/plate/1', 'name': 'Plate 1'}, ...] - await self.compilation_service.compile_plates(selected) - -Main Methods -~~~~~~~~~~~~ - -.. code-block:: python - - async def compile_plates(self, selected_items: List[Dict]) -> None: - """ - Compile pipelines for selected plates. - - Args: - selected_items: List of plate data dicts with 'path' and 'name' keys - """ - -Compilation Flow ----------------- - -1. **Context Setup** - Ensures global config context is available in worker thread -2. **Progress Initialization** - Calls ``emit_progress_started(count)`` -3. **Per-Plate Compilation**: - - - Get pipeline definition from host - - Validate step func attributes - - Get or create orchestrator - - Initialize pipeline - - Compile with variable components - - Update progress - -4. **Progress Completion** - Calls ``emit_progress_finished()`` - -Internal Methods ----------------- - -.. code-block:: python - - async def _get_or_create_orchestrator(self, plate_path: str) -> PipelineOrchestrator: - """Get existing orchestrator or create new one.""" - - def _validate_pipeline_steps(self, steps: List) -> None: - """Validate that all steps have func attributes.""" - - async def _initialize_and_compile(self, orchestrator, steps, plate_data) -> None: - """Initialize pipeline and compile with variable components.""" - -Error Handling --------------- - -Compilation errors are reported via the host's ``emit_compilation_error`` callback: - -.. code-block:: python - - try: - await self._initialize_and_compile(orchestrator, steps, plate_data) - except Exception as e: - self.host.emit_compilation_error(plate_data['name'], str(e)) - logger.exception(f"Compilation failed for {plate_data['name']}") - -Integration with PlateManager ------------------------------ - -In ``PlateManagerWidget``: - -.. code-block:: python - - class PlateManagerWidget(AbstractManagerWidget, CompilationHost): - def __init__(self): - super().__init__() - self._compilation_service = CompilationService(host=self) - - # CompilationHost protocol implementation - def emit_progress_started(self, count: int) -> None: - self.progress_bar.setMaximum(count) - self.progress_bar.show() - - def emit_compilation_error(self, plate_name: str, error: str) -> None: - self.error_log.append(f"❌ {plate_name}: {error}") - - async def action_compile(self): - """Compile selected plates.""" - selected = self._get_selected_plate_items() - await self._compilation_service.compile_plates(selected) - -See Also +Also see -------- -- :doc:`zmq_execution_service_extracted` - Execution service for running compiled pipelines -- :doc:`abstract_manager_widget` - ABC that PlateManager inherits from -- :doc:`plate_manager_services` - Other PlateManager service extractions -- :doc:`pipeline_compilation_system` - Core pipeline compilation architecture - +- :doc:`plate_manager_services` +- :doc:`progress_runtime_projection_system` diff --git a/docs/source/architecture/compilation_system_detailed.rst b/docs/source/architecture/compilation_system_detailed.rst index 2127da39a..f83fe5b9b 100644 --- a/docs/source/architecture/compilation_system_detailed.rst +++ b/docs/source/architecture/compilation_system_detailed.rst @@ -35,7 +35,10 @@ Compilation Flow Summary **Key Insight**: Function patterns are modified during compilation (metadata injection) and stored in ``step_plans['func']`` by the -FuncStepContractValidator, then retrieved during execution. +FuncStepContractValidator. The compiler returns a ``step_state_map`` +that lets downstream validators and streaming helpers read the +the saved configuration snapshot for each step so that only the +compile-time values (not live UI edits) influence execution plans. Phase-by-Phase Detailed Flow ---------------------------- @@ -48,14 +51,26 @@ Entry Point: ``PipelineOrchestrator.compile_pipelines()`` for well_id in wells_to_process: context = self.create_context(well_id) - # 5-Phase Compilation (actual implementation) - # All phases wrapped in config_context for lazy resolution - with config_context(orchestrator.pipeline_config): - PipelineCompiler.initialize_step_plans_for_context(context, pipeline_definition, orchestrator, metadata_writer=is_responsible, plate_path=orchestrator.plate_path) - PipelineCompiler.declare_zarr_stores_for_context(context, pipeline_definition, orchestrator) - PipelineCompiler.plan_materialization_flags_for_context(context, pipeline_definition, orchestrator) - PipelineCompiler.validate_memory_contracts_for_context(context, pipeline_definition, orchestrator) - PipelineCompiler.assign_gpu_resources_for_context(context) + # 5-Phase Compilation (actual implementation) + # All phases wrapped in config_context for lazy resolution + with config_context(orchestrator.pipeline_config): + resolved_steps, step_state_map = PipelineCompiler.initialize_step_plans_for_context( + context, + pipeline_definition, + orchestrator, + metadata_writer=is_responsible, + plate_path=orchestrator.plate_path, + steps_already_resolved=False, + ) + PipelineCompiler.declare_zarr_stores_for_context(context, resolved_steps, orchestrator) + PipelineCompiler.plan_materialization_flags_for_context(context, resolved_steps, orchestrator) + PipelineCompiler.validate_memory_contracts_for_context( + context, + resolved_steps, + orchestrator, + step_state_map=step_state_map, + ) + PipelineCompiler.assign_gpu_resources_for_context(context, resolved_steps, orchestrator) context.freeze() compiled_contexts[well_id] = context @@ -63,29 +78,46 @@ Entry Point: ``PipelineOrchestrator.compile_pipelines()`` Phase 1: Step Plan Initialization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -**File**: ``openhcs/core/pipeline/compiler.py:229-282`` +**File**: ``openhcs/core/pipeline/compiler.py:229-360`` .. code:: python - def initialize_step_plans_for_context(context, steps_definition, orchestrator, metadata_writer=False, plate_path=None): - # Pre-initialize basic step_plans using step index as key - for step_index, step in enumerate(steps_definition): - context.step_plans[step_index] = { - "step_name": step.name, - "step_type": step.__class__.__name__, - "axis_id": context.axis_id, + def initialize_step_plans_for_context( + context, + steps_definition, + orchestrator, + metadata_writer=False, + plate_path=None, + step_state_map=None, + steps_already_resolved=True, + ): + # Pre-initialize basic step_plans using step index as key + for step_index, step in enumerate(steps_definition): + context.step_plans[step_index] = { + "step_name": step.name, + "step_type": step.__class__.__name__, + "axis_id": context.axis_id, } - # Call path planner - THIS IS WHERE METADATA INJECTION HAPPENS - PipelinePathPlanner.prepare_pipeline_paths(context, steps_definition, orchestrator.pipeline_config) - - # Post-path-planner processing (stores func_name but NOT func) - for step_index, step in enumerate(steps_definition): - if isinstance(step, FunctionStep): - current_plan = context.step_plans[step_index] - if hasattr(step, 'func'): - current_plan["func_name"] = getattr(step.func, '__name__', str(step.func)) - # NOTE: step.func is NOT stored here - happens in Phase 4 + # Register ObjectState layers (global → orchestrator → step) + # and resolve the steps_before_path_planner using saved values. + PipelinePathPlanner.prepare_pipeline_paths( + context, + steps_definition, + orchestrator.pipeline_config, + ) + + # Post-path-planner processing (stores func_name but NOT func) + for step_index, step in enumerate(steps_definition): + if isinstance(step, FunctionStep): + current_plan = context.step_plans[step_index] + if hasattr(step, 'func'): + current_plan["func_name"] = getattr(step.func, '__name__', str(step.func)) + # NOTE: step.func is NOT stored here - happens in Phase 4 + + # Return the resolved steps and the registered ObjectState map so later + # planner phases can access saved configuration values deterministically. + return steps_definition, step_state_map Critical Sub-Phase: Metadata Injection in Path Planner ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/architecture/declarative_window_system.rst b/docs/source/architecture/declarative_window_system.rst new file mode 100644 index 000000000..64f75be89 --- /dev/null +++ b/docs/source/architecture/declarative_window_system.rst @@ -0,0 +1,406 @@ +Declarative Window System +========================= + +**Centralized window management with WindowManager integration and declarative specifications.** + +*Status: STABLE* +*Module: openhcs.pyqt_gui* + +Overview +-------- + +The Declarative Window System provides a **clean, centralized approach** to managing application windows in OpenHCS PyQt6 GUI. It replaces the ad-hoc floating window dictionary pattern with a declarative specification system integrated with the WindowManager singleton. + +**The Problem**: The original implementation used a ``floating_windows: Dict[str, QDialog]`` dictionary in ``main.py`` with hardcoded window creation logic scattered throughout. This created tight coupling between the main window and window implementations, making it difficult to: + +- Track window lifecycle consistently +- Ensure singleton-per-scope behavior +- Share window management patterns across the application +- Test window creation in isolation + +**The Solution**: A declarative specification pattern where: + +1. **WindowSpec** defines window configuration (ID, title, class, initialization behavior) +2. **ManagedWindow** classes encapsulate window setup and signal connections +3. **WindowManager** handles singleton enforcement and lifecycle management +4. **MainWindow** uses declarative specs instead of hardcoded creation logic + +Key Architectural Patterns +-------------------------- + +WindowSpec Declaration Pattern +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Problem**: Hardcoded window creation scattered across main.py. + +**Solution**: Centralized declarative specifications in ``_get_window_specs()``. + +.. code-block:: python + + def _get_window_specs(self) -> dict[str, WindowSpec]: + """Return declarative window specifications.""" + return { + "plate_manager": WindowSpec( + window_id="plate_manager", + title="Plate Manager", + window_class=PlateManagerWindow, + initialize_on_startup=True, + ), + "pipeline_editor": WindowSpec( + window_id="pipeline_editor", + title="Pipeline Editor", + window_class=PipelineEditorWindow, + ), + # ... more windows + } + +**Key Points**: + +- Window configuration is **data**, not code +- Window ID is the canonical identifier +- Window class encapsulates all setup logic +- ``initialize_on_startup`` for windows needed immediately + +ManagedWindow Encapsulation Pattern +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Problem**: Window setup logic mixed with main window concerns. + +**Solution**: Each window type has its own ManagedWindow class. + +.. code-block:: python + + class PlateManagerWindow(QDialog): + """Window wrapper for PlateManagerWidget.""" + + def __init__(self, main_window, service_adapter): + super().__init__(main_window) + self.main_window = main_window + self.service_adapter = service_adapter + self.setWindowTitle("Plate Manager") + self.setModal(False) + self.resize(600, 400) + + # Setup UI + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + layout = QVBoxLayout(self) + self.widget = PlateManagerWidget(service_adapter, ...) + layout.addWidget(self.widget) + + # Setup connections + self._setup_connections() + + def _setup_connections(self): + """Connect signals to main window and other windows.""" + self.widget.global_config_changed.connect(...) + # ... more connections + +**Key Points**: + +- Window is responsible for its own setup +- Main window only needs to call constructor +- Signal connections encapsulated in the window class +- Easy to test in isolation + +WindowManager Integration Pattern +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Problem**: Manual singleton enforcement and window tracking. + +**Solution**: Delegate to WindowManager singleton service. + +.. code-block:: python + + def show_window(self, window_id: str) -> None: + """Show window using WindowManager.""" + factory = self._create_window_factory(window_id) + window = WindowManager.show_or_focus(window_id, factory) + self._ensure_flash_overlay(window) + + def _create_window_factory(self, window_id: str) -> Callable[[], QDialog]: + """Create factory function for a window.""" + spec = self.window_specs[window_id] + + def factory() -> QDialog: + return spec.window_class(self, self.service_adapter) + + return factory + +**Key Points**: + +- Factory pattern defers creation until needed +- WindowManager enforces singleton-per-scope +- Flash overlay integration for visual feedback +- No manual window tracking needed + +Migration from Floating Windows +-------------------------------- + +Before: Floating Window Dictionary +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + :caption: Before: Hardcoded creation in main.py + + class OpenHCSMainWindow(QMainWindow): + def __init__(self): + super().__init__() + self.floating_windows: Dict[str, QDialog] = {} + + def show_plate_manager(self): + if "plate_manager" not in self.floating_windows: + window = QDialog(self) + window.setWindowTitle("Plate Manager") + window.setModal(False) + window.resize(600, 400) + + layout = QVBoxLayout(window) + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + widget = PlateManagerWidget(self.service_adapter, ...) + layout.addWidget(widget) + + # Setup connections inline (messy!) + widget.global_config_changed.connect(...) + widget.progress_started.connect(...) + # ... many more connections + + self.floating_windows["plate_manager"] = window + + self.floating_windows["plate_manager"].show() + self.floating_windows["plate_manager"].raise_() + self.floating_windows["plate_manager"].activateWindow() + +**Problems**: + +- Window creation logic scattered across methods +- Signal connections mixed with UI setup +- Manual window tracking dictionary +- No consistent singleton enforcement +- Hard to test (requires full main window) + +After: Declarative Specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + :caption: After: Declarative specs with WindowManager + + class OpenHCSMainWindow(QMainWindow): + def __init__(self): + super().__init__() + self.window_specs = self._get_window_specs() + + def show_window(self, window_id: str) -> None: + """Show any window using WindowManager.""" + factory = self._create_window_factory(window_id) + window = WindowManager.show_or_focus(window_id, factory) + self._ensure_flash_overlay(window) + + def show_plate_manager(self): + """Show plate manager - delegates to WindowManager.""" + self.show_window("plate_manager") + +**Benefits**: + +- One method shows any window +- WindowManager handles singleton enforcement +- Window classes handle their own setup +- Easy to add new window types (just add to specs) +- Testable: window classes can be tested without main window + +Window Configuration Service +----------------------------- + +The ``window_config.py`` module provides the data structures for declarative window specifications. + +WindowSpec Dataclass +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + @dataclass(frozen=True) + class WindowSpec: + """ + Declarative specification for an application window. + + Centralizes window configuration (widget, title, initialization behavior). + """ + window_id: str # Unique identifier (e.g., "plate_manager") + title: str # Window title shown to user + window_class: Type[ManagedWindow] # Class to instantiate + initialize_on_startup: bool = False # Create on startup (hidden)? + +ManagedWindow Base Class +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + class ManagedWindow(QDialog): + """Base class for managed application windows.""" + + def __init__(self, main_window, service_adapter): + super().__init__(main_window) + self.main_window = main_window + self.service_adapter = service_adapter + self.setup_ui() + self.setup_connections() + + def setup_ui(self): + """Setup window UI. Subclasses implement this.""" + pass + + def setup_connections(self): + """Setup signal connections. Subclasses implement this.""" + pass + +Window Implementations +---------------------- + +The ``managed_windows.py`` module contains all ManagedWindow implementations. + +PlateManagerWindow +~~~~~~~~~~~~~~~~~~~ + +Wraps ``PlateManagerWidget`` with proper signal connections to main window and pipeline editor. + +.. code-block:: python + + class PlateManagerWindow(QDialog): + def __init__(self, main_window, service_adapter): + # ... setup code ... + self._setup_connections() + + def _setup_connections(self): + # Connect to main window + self.widget.global_config_changed.connect( + lambda: self.main_window.on_config_changed(...) + ) + + # Connect progress signals to status bar + if hasattr(self.main_window, "status_bar"): + self._setup_progress_signals() + + # Connect to pipeline editor (bidirectional) + self._connect_to_pipeline_editor() + +PipelineEditorWindow +~~~~~~~~~~~~~~~~~~~~ + +Wraps ``PipelineEditorWidget`` with connections to plate manager. + +ImageBrowserWindow +~~~~~~~~~~~~~~~~~~~ + +Wraps ``ImageBrowserWidget`` with orchestrator updates from plate selection. + +LogViewerWindowWrapper +~~~~~~~~~~~~~~~~~~~~~~~ + +Wraps ``LogViewerWindow`` for log file viewing. + +ZMQServerManagerWindow +~~~~~~~~~~~~~~~~~~~~~~~ + +Wraps ``ZMQServerManagerWidget`` for ZMQ server monitoring. + +Integration with BaseFormDialog +-------------------------------- + +Dialog windows (ConfigWindow, DualEditorWindow, etc.) inherit from ``BaseFormDialog`` from the pyqt-reactive submodule, which provides: + +- **Singleton-per-scope behavior**: Only one window per scope_id +- **ObjectState integration**: Automatic save/restore on accept/reject +- **WindowManager registration**: Automatic lifecycle management +- **Change detection**: Optional automatic change detection support + +.. code-block:: python + + from pyqt_reactive.widgets.shared import BaseFormDialog + + class ConfigWindow(BaseFormDialog): + def __init__(self, config_class, initial_config, parent, scope_id): + super().__init__(parent) + self.scope_id = scope_id # Enables singleton behavior + # ... setup ... + +Benefits and Impact +-------------------- + +Code Reduction +~~~~~~~~~~~~~~ + +**Eliminated**: + +- ~200 lines of hardcoded window creation in main.py +- Floating window dictionary management +- Manual singleton enforcement logic +- Duplicated window setup patterns + +**Added**: + +- ~50 lines in window_config.py (one-time) +- ~150 lines in managed_windows.py (all windows) +- **Net reduction**: ~100+ lines + +Maintainability +~~~~~~~~~~~~~~~~ + +**Single Source of Truth**: Window specifications are data, not code. + +- Add new window: Add one entry to ``_get_window_specs()`` +- Modify window: Edit one class in ``managed_windows.py`` +- Remove window: Delete one spec and one class + +**Testability**: Window classes can be tested independently. + +.. code-block:: python + + # Test without full main window + window = PlateManagerWindow(mock_main_window, mock_service_adapter) + assert window.windowTitle() == "Plate Manager" + # Test signal connections... + +Consistency +~~~~~~~~~~~~ + +All windows now follow the same pattern: + +1. **ManagedWindow** for simple windows (PlateManager, etc.) +2. **BaseFormDialog** for config dialogs (from pyqt-reactive) +3. **WindowManager** for singleton enforcement (all windows) + +Extensibility +~~~~~~~~~~~~~~ + +**Adding a new window type**: + +1. Create ``MyWindow(QDialog)`` class in ``managed_windows.py`` +2. Add spec to ``_get_window_specs()`` in ``main.py`` +3. Done! WindowManager handles the rest. + +Related Patterns +----------------- + +**See Also**: + +- :doc:`window_manager_usage` - WindowManager API reference +- :doc:`service-layer-architecture` - Service layer patterns +- :doc:`ui_services_architecture` - UI service patterns + +**Integration Points**: + +- **WindowManager** (pyqt-reactive): Singleton window registry +- **BaseFormDialog** (pyqt-reactive): Base class for config dialogs +- **ObjectState**: State management for config dialogs +- **ServiceAdapter**: Provides services to windows + +Summary +------- + +The Declarative Window System demonstrates how moving from imperative window creation to declarative specifications improves: + +- **Maintainability**: Centralized configuration, clear separation of concerns +- **Testability**: Window classes testable in isolation +- **Consistency**: All windows follow the same patterns +- **Extensibility**: New windows require minimal boilerplate + +By leveraging the WindowManager singleton and BaseFormDialog from pyqt-reactive, OpenHCS achieves robust window management with minimal custom code. diff --git a/docs/source/architecture/dynamic_dataclass_factory.rst b/docs/source/architecture/dynamic_dataclass_factory.rst index 9c713cff8..4034a82cd 100644 --- a/docs/source/architecture/dynamic_dataclass_factory.rst +++ b/docs/source/architecture/dynamic_dataclass_factory.rst @@ -60,6 +60,24 @@ Generated classes are automatically registered for type mapping. The registry is populated automatically when :py:meth:`~openhcs.config_framework.lazy_factory.LazyDataclassFactory.make_lazy_simple` creates a new lazy class. You can retrieve the base type using :py:func:`~openhcs.config_framework.lazy_factory.get_base_type_for_lazy`. +Dataclass Reconstruction (Serialization) +-------------------------------------- + +OpenHCS uses the ObjectState lazy framework for config/lazy dataclass resolution and serialization. +When resolved lazy dataclasses are converted back into concrete dataclasses (e.g. for pickling), +ObjectState reconstructs objects from a resolved ``{field_name: value}`` mapping. + +If a dataclass uses a custom constructor (common with ``@dataclass(init=False)``), reconstruction +via ``Type(**fields)`` may fail. In that case, the dataclass should provide a rebuild hook: + +.. code-block:: python + + @classmethod + def __objectstate_rebuild__(cls, **fields): + return cls(*fields["outputs"], primary=fields.get("primary", 0)) + +This keeps custom constructors clean while still supporting lazy resolution and serialization. + Dual-Axis Resolution Strategy ----------------------------- The factory uses a two-axis resolution strategy to find field values. diff --git a/docs/source/architecture/external_integrations_overview.rst b/docs/source/architecture/external_integrations_overview.rst index 5a6374487..49362b1e3 100644 --- a/docs/source/architecture/external_integrations_overview.rst +++ b/docs/source/architecture/external_integrations_overview.rst @@ -13,6 +13,24 @@ OpenHCS implements a comprehensive integration strategy with the bioimage analys This document provides a high-level overview of OpenHCS's integration architecture and how the different components work together to create a unified bioimage analysis platform. +Boundary Split +-------------- + +The integration stack is now explicitly split across repositories: + +- ``zmqruntime`` owns transport and execution lifecycle primitives + (REQ/REP + PUB/SUB channels, polling, typed execution status contracts). +- ``polystore`` owns streaming payload semantics and receiver-side projection + utilities (component-mode grouping, batch/debounce engines, viewer payload + constants). +- ``pyqt-reactive`` owns generic manager/browser UI infrastructure + (polling shells, tree sync/rebuild, aggregation policies). +- ``openhcs`` owns domain wiring only + (pipeline/orchestrator semantics, OpenHCS-specific adapters, UX policies). + +OpenHCS architecture docs focus on OpenHCS wrapper behavior and link to +external module docs for generic abstraction internals. + Integration Philosophy ---------------------- @@ -60,7 +78,8 @@ Napari Integration **Purpose**: Real-time visualization of processing results -**Architecture**: Streaming backend + persistent viewer processes +**Architecture**: OpenHCS wrapper over ``zmqruntime`` transport plus +``polystore`` streaming semantics and receiver projection **Key Features**: @@ -108,7 +127,8 @@ Fiji Integration **Purpose**: Interoperability with ImageJ/Fiji ecosystem -**Architecture**: Streaming backend + PyImageJ integration +**Architecture**: OpenHCS wrapper over ``zmqruntime`` transport plus +``polystore`` streaming semantics and receiver projection **Key Features**: diff --git a/docs/source/architecture/fiji_streaming_system.rst b/docs/source/architecture/fiji_streaming_system.rst index 32f15d202..ba940599c 100644 --- a/docs/source/architecture/fiji_streaming_system.rst +++ b/docs/source/architecture/fiji_streaming_system.rst @@ -8,7 +8,20 @@ Pipeline visualization with Fiji/ImageJ requires real-time data streaming to ext **The Fiji Integration Challenge**: ImageJ/Fiji uses a different dimensional model (CZT: Channels, Z-slices, Time-frames) than OpenHCS's component-based system. Additionally, hyperstack building is computationally expensive (~2 seconds per stack), which could block pipeline execution if not handled properly. -**The OpenHCS Solution**: A process-based streaming architecture that separates Fiji visualization into independent processes communicating via ZeroMQ. This eliminates blocking issues while enabling automatic hyperstack creation from OpenHCS's component metadata. +**The OpenHCS Solution**: A thin-wrapper architecture where OpenHCS composes +external streaming infrastructure. Generic transport is provided by +``zmqruntime``, payload semantics and receiver-core utilities are provided by +``polystore``, and OpenHCS provides runtime wiring and process management. + +Canonical Abstraction Docs +-------------------------- + +For generic abstraction internals, see external module docs: + +- ``external/zmqruntime/docs/source/architecture/viewer_streaming_architecture.rst`` +- ``external/PolyStore/docs/source/architecture/streaming_receiver_projection.rst`` + +This page focuses on OpenHCS-specific viewer wrapper behavior. **Key Innovation**: Shared memory IPC with proper lifecycle management ensures zero-copy data transfer while preventing memory leaks. The publisher closes handles after successful sends, while the receiver unlinks shared memory after copying data. @@ -18,7 +31,7 @@ Architecture Components FijiStreamingBackend ~~~~~~~~~~~~~~~~~~~~ -**Location**: ``openhcs/io/fiji_stream.py`` +**Location**: ``external/PolyStore/src/polystore/fiji_stream.py`` The streaming backend sends image data to Fiji viewers using ZeroMQ publish/subscribe pattern: @@ -58,7 +71,7 @@ The streaming backend sends image data to Fiji viewers using ZeroMQ publish/subs FijiViewerServer ~~~~~~~~~~~~~~~~ -**Location**: ``openhcs/runtime/fiji_viewer_server.py`` +**Location**: ``openhcs/runtime/fiji_viewer_server.py`` (OpenHCS wrapper) The viewer server receives images and displays them via PyImageJ, inheriting from ``ZMQServer`` ABC: @@ -94,7 +107,7 @@ The viewer server receives images and displays them via PyImageJ, inheriting fro FijiStreamVisualizer ~~~~~~~~~~~~~~~~~~~~ -**Location**: ``openhcs/runtime/fiji_stream_visualizer.py`` +**Location**: ``openhcs/runtime/fiji_stream_visualizer.py`` (OpenHCS wrapper) Manages Fiji viewer process lifecycle, following the same architecture as ``NapariStreamVisualizer``: diff --git a/docs/source/architecture/index.rst b/docs/source/architecture/index.rst index b4d217d18..7a4605aaf 100644 --- a/docs/source/architecture/index.rst +++ b/docs/source/architecture/index.rst @@ -22,6 +22,7 @@ Fundamental systems that define OpenHCS architecture. custom_function_registration_system pipeline_compilation_system special_io_system + pattern_grouping_and_special_outputs roi_system analysis_consolidation_system experimental_analysis_system @@ -63,6 +64,7 @@ Integration with external tools and platforms (Napari, OMERO, Fiji). :maxdepth: 1 external_integrations_overview + streaming_boundary_and_wrappers napari_integration_architecture omero_backend_system fiji_streaming_system @@ -117,7 +119,7 @@ Dynamic code generation and parser systems. pattern_detection_system User Interface Systems -====================== +===================== TUI architecture, UI development patterns, and form management systems. @@ -132,17 +134,20 @@ TUI architecture, UI development patterns, and form management systems. flash_animation_system scope_visual_feedback_system plate_manager_services + batch_workflow_service + progress_runtime_projection_system + zmq_server_browser_system parameter_form_lifecycle parameter_form_service_architecture ui_services_architecture field_change_dispatcher parametric_widget_creation - compilation_service - zmq_execution_service_extracted code_ui_interconversion service-layer-architecture gui_performance_patterns cross_window_update_optimization + scope_window_factory_system + service_registry_integration Development Tools ================= @@ -167,7 +172,7 @@ Quick Start Paths **External Integrations?** Start with :doc:`external_integrations_overview` → :doc:`napari_integration_architecture` → :doc:`fiji_streaming_system` → :doc:`omero_backend_system` -**UI Development?** Start with :doc:`widget_protocol_system` → :doc:`abstract_manager_widget` → :doc:`parametric_widget_creation` → :doc:`field_change_dispatcher` → :doc:`ui_services_architecture` → :doc:`compilation_service` → :doc:`tui_system` +**UI Development?** Start with :doc:`widget_protocol_system` → :doc:`abstract_manager_widget` → :doc:`parametric_widget_creation` → :doc:`field_change_dispatcher` → :doc:`ui_services_architecture` → :doc:`batch_workflow_service` → :doc:`zmq_server_browser_system` → :doc:`tui_system` **System Integration?** Jump to :doc:`system_integration` → :doc:`special_io_system` → :doc:`microscope_handler_integration` diff --git a/docs/source/architecture/napari_streaming_system.rst b/docs/source/architecture/napari_streaming_system.rst index a74391f4d..3174d2471 100644 --- a/docs/source/architecture/napari_streaming_system.rst +++ b/docs/source/architecture/napari_streaming_system.rst @@ -8,7 +8,21 @@ Pipeline visualization requires real-time data streaming to external processes w **The Visualization Challenge**: Traditional visualization approaches embed viewers in the main process, causing Qt threading conflicts and blocking pipeline execution. This creates a fundamental tension between visualization needs and processing performance. -**The OpenHCS Solution**: A process-based streaming architecture that separates visualization into independent processes communicating via ZeroMQ. This eliminates Qt threading issues while enabling true real-time monitoring without performance impact on pipeline execution. +**The OpenHCS Solution**: A thin-wrapper composition architecture. OpenHCS uses +``zmqruntime`` for transport/lifecycle and ``polystore`` for streaming payload +semantics and receiver-side batching/projection, while OpenHCS runtime modules +provide application-specific process management and orchestration wiring. + +Canonical Abstraction Docs +-------------------------- + +For generic abstraction internals, see external module docs: + +- ``external/zmqruntime/docs/source/architecture/zmq_execution_system.rst`` +- ``external/zmqruntime/docs/source/architecture/viewer_streaming_architecture.rst`` +- ``external/PolyStore/docs/source/architecture/streaming_receiver_projection.rst`` + +This page focuses on OpenHCS wrapper behavior and integration decisions. **Key Innovation**: Materialization-aware filtering ensures only meaningful outputs (final results, checkpoints) are visualized rather than overwhelming users with every intermediate processing step. diff --git a/docs/source/architecture/pattern_detection_system.rst b/docs/source/architecture/pattern_detection_system.rst index f2e34034f..3c50497a8 100644 --- a/docs/source/architecture/pattern_detection_system.rst +++ b/docs/source/architecture/pattern_detection_system.rst @@ -11,6 +11,11 @@ The Solution: Automatic Pattern Discovery OpenHCS implements a pattern detection system that automatically discovers image file patterns across different microscope formats. This system coordinates filename parsing, directory structure analysis, and pattern grouping to enable flexible pipeline processing without manual configuration. +.. seealso:: + + :doc:`pattern_grouping_and_special_outputs` + Detailed explanation of how pattern grouping interacts with special outputs and the dual purpose of ``group_by`` + Overview -------- diff --git a/docs/source/architecture/pattern_grouping_and_special_outputs.rst b/docs/source/architecture/pattern_grouping_and_special_outputs.rst new file mode 100644 index 000000000..d003a66a9 --- /dev/null +++ b/docs/source/architecture/pattern_grouping_and_special_outputs.rst @@ -0,0 +1,434 @@ +Pattern Grouping and Special Output Path Resolution +==================================================== + +Overview +-------- + +This document explains the fundamental principles of how OpenHCS handles pattern grouping, special output path resolution, and the interaction between ``group_by``, ``variable_components``, and the special I/O system. Understanding these principles is essential for debugging path collisions and understanding compilation behavior. + +**Critical Insight**: The ``group_by`` parameter serves dual purposes: +1. For **dict patterns**: Specifies what dimension the dictionary keys represent +2. For **list patterns**: Controls pattern grouping and special output namespacing + +This dual purpose is intentional and enables compile-time path planning with deterministic, semantic paths. + +First Principles +---------------- + +Pattern Types and Internal Representation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +OpenHCS internally normalizes all function patterns to a dictionary format for uniform processing: + +**Dict Patterns** (explicit): + +.. code-block:: python + + # User writes: + FunctionStep( + func={'1': analyze_nuclei, '2': analyze_gfp}, + group_by=GroupBy.CHANNEL + ) + + # Internal representation: (unchanged) + {'1': analyze_nuclei, '2': analyze_gfp} + +**List Patterns** (normalized to dict): + +.. code-block:: python + + # User writes: + FunctionStep(func=[normalize, tophat]) + + # Internal representation: + {'default': [normalize, tophat]} + +**Single Patterns** (normalized to dict): + +.. code-block:: python + + # User writes: + FunctionStep(func=(normalize, {})) + + # Internal representation: + {'default': (normalize, {})} + +**The "default" Key Convention**: +- String ``"default"`` is used as the internal dict key for non-dict patterns +- Converted to ``None`` when used as a special output group key +- See ``openhcs/formats/func_arg_prep.py::iter_pattern_items()`` and ``openhcs/core/pipeline/path_planner.py::extract_attributes()`` + +The Role of ``group_by`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**For Dict Patterns**: Semantic Meaning of Keys +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``group_by`` parameter tells OpenHCS what dimension the dictionary keys represent: + +.. code-block:: python + + # Keys are channel numbers + FunctionStep( + func={'1': analyze_nuclei, '2': analyze_gfp}, + group_by=GroupBy.CHANNEL # Keys '1' and '2' are channel numbers + ) + + # Keys are well identifiers + FunctionStep( + func={'control': process_control, 'treatment': process_treatment}, + group_by=GroupBy.WELL # Keys are well names + ) + +**For List Patterns**: Pattern Grouping and Output Namespacing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For list patterns, ``group_by`` controls how patterns are organized during discovery and how special outputs are namespaced: + +.. code-block:: python + + # WITHOUT group_by: All patterns processed together + FunctionStep( + func=(normalize, {}), + group_by=None, # or GroupBy.NONE + variable_components=[VariableComponents.SITE, VariableComponents.CHANNEL] + ) + # Pattern discovery returns: + # {"default": ["A01_s{iii}_w1_z001.tif", "A01_s{iii}_w2_z001.tif"]} + # Special outputs: All patterns write to SAME path → COLLISION! + + # WITH group_by: Patterns grouped by component + FunctionStep( + func=(normalize, {}), + group_by=GroupBy.CHANNEL, + variable_components=[VariableComponents.SITE, VariableComponents.CHANNEL] + ) + # Pattern discovery returns: + # {"1": ["A01_s{iii}_w1_z001.tif"], "2": ["A01_s{iii}_w2_z001.tif"]} + # Special outputs: Each channel gets its own path → NO COLLISION! + +The Dual Purpose of ``group_by`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Why does ``group_by`` affect list patterns?** + +The system uses ``group_by`` for list patterns to enable: + +1. **Semantic Grouping**: Patterns are organized by meaningful component values (channel 1, channel 2) rather than arbitrary indices +2. **Deterministic Paths**: Special output paths are known at compile time, not runtime +3. **Cross-Step Communication**: Later steps can reference special outputs by component (e.g., "get cell_counts for channel 1") +4. **Compile-Time Validation**: Path collisions are detected during compilation, not execution + +**Alternative Considered**: Runtime collision detection with auto-generated suffixes + +.. code-block:: python + + # Runtime collision detection (NOT IMPLEMENTED): + if vfs_path in already_saved_paths: + vfs_path = f"{vfs_path.stem}_pattern{index}{vfs_path.suffix}" + + # Problems: + # 1. Non-deterministic paths (unknown until runtime) + # 2. Cross-step communication breaks (can't reference by name) + # 3. Loss of semantic meaning (cell_counts_pattern0.pkl vs cell_counts_w1.pkl) + +**Conclusion**: Using ``group_by`` for list patterns provides compile-time guarantees and semantic clarity. + +Pattern Discovery and Grouping Flow +------------------------------------ + +Understanding the complete flow from pattern discovery to execution is essential for debugging path issues. + +Step 1: Pattern Discovery +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``PatternDiscoveryEngine`` discovers patterns based on ``variable_components``: + +.. code-block:: python + + # Configuration: + variable_components = [VariableComponents.SITE, VariableComponents.CHANNEL] + group_by = GroupBy.CHANNEL + + # Files in directory: + # A01_s001_w1_z001_t001.tif + # A01_s001_w2_z001_t001.tif + # A01_s002_w1_z001_t001.tif + # A01_s002_w2_z001_t001.tif + + # Step 1: Generate patterns (based on variable_components) + patterns = [ + "A01_s{iii}_w1_z001_t001.tif", # Site varies, channel=1 fixed + "A01_s{iii}_w2_z001_t001.tif" # Site varies, channel=2 fixed + ] + + # Step 2: Group patterns (based on group_by) + if group_by == GroupBy.CHANNEL: + # Parse each pattern to extract channel value + # "A01_s{iii}_w1_z001_t001.tif" → replace {iii} with 001 → parse → channel='1' + grouped_patterns = { + "1": ["A01_s{iii}_w1_z001_t001.tif"], + "2": ["A01_s{iii}_w2_z001_t001.tif"] + } + else: + # No grouping + grouped_patterns = ["A01_s{iii}_w1_z001_t001.tif", "A01_s{iii}_w2_z001_t001.tif"] + +**Key Files**: +- ``openhcs/formats/pattern/pattern_discovery.py::auto_detect_patterns()`` (line 233-277) +- ``openhcs/formats/pattern/pattern_discovery.py::group_patterns_by_component()`` (line 157-202) + +Step 2: Path Planning (Compilation) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``PathPlanner`` determines execution groups and creates special output paths: + +.. code-block:: python + + # For list patterns with group_by: + def _get_execution_groups(step, step_index): + # Resolve group_by via ObjectState (handles lazy dataclasses) + group_by = step_state.get_saved_resolved_value("processing_config.group_by") + + if group_by and group_by != GroupBy.NONE: + # Get component keys from orchestrator + return ["1", "2"] # For CHANNEL with 2 channels + else: + return [None] # No grouping + + # Create special output paths for each group: + execution_groups = ["1", "2"] + for output_key in special_outputs: + paths_by_group = { + "1": "A01_w1_cell_counts_step7.pkl", + "2": "A01_w2_cell_counts_step7.pkl" + } + +**Critical**: The path planner must use ``ObjectState.get_saved_resolved_value()`` to resolve ``group_by`` from lazy dataclasses, NOT direct ``getattr()`` access. + +**Key Files**: +- ``openhcs/core/pipeline/path_planner.py::_get_execution_groups()`` (line 105-146) +- ``openhcs/core/pipeline/path_planner.py::_build_paths_by_group()`` (line 145-157) +- ``openhcs/core/pipeline/compiler.py::initialize_step_plans_for_context()`` (line 495-505) + +Step 3: Pattern Preparation (Execution) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``prepare_patterns_and_functions()`` normalizes patterns to dict format: + +.. code-block:: python + + # Input from pattern discovery: + patterns = {"1": ["w1_pattern"], "2": ["w2_pattern"]} # Already grouped + func = (normalize, {}) # List pattern + + # Normalization: + grouped_patterns = patterns # Already a dict, use as-is + component_to_funcs = { + "1": [(normalize, {})], # Same function for channel 1 + "2": [(normalize, {})] # Same function for channel 2 + } + +**Key Files**: +- ``openhcs/formats/func_arg_prep.py::prepare_patterns_and_functions()`` (line 96-273) + +Step 4: Execution Loop +~~~~~~~~~~~~~~~~~~~~~~ + +The execution loop processes each component group separately: + +.. code-block:: python + + # For each component value: + for comp_val, pattern_list in grouped_patterns.items(): + # comp_val = "1" or "2" + exec_func = component_to_funcs[comp_val] + + # For each pattern in this component group: + for pattern in pattern_list: + _process_single_pattern_group( + ..., + component_value=comp_val, # "1" or "2" + ... + ) + + # Inside _process_single_pattern_group: + component_key = str(component_value) # "1" or "2" + + # Select special outputs for this component: + special_outputs_for_component = _select_special_plan_for_component( + special_outputs_by_group, # {"1": {...}, "2": {...}} + component_key, # "1" or "2" + default_plan + ) + # Returns: {"cell_counts": {"path": "A01_w1_cell_counts_step7.pkl"}} + +**Key Files**: +- ``openhcs/core/steps/function_step.py::process()`` (line 1316-1356) +- ``openhcs/core/steps/function_step.py::_process_single_pattern_group()`` (line 701-900) +- ``openhcs/core/steps/function_step.py::_select_special_plan_for_component()`` (line 78-90) + +Common Issues and Debugging +---------------------------- + +Issue 1: Special Output Path Collision with List Patterns +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Symptom**: + +.. code-block:: text + + FileExistsError: Path already exists: /path/to/results/A01_cell_counts_step7.pkl + +**Root Cause**: Multiple patterns trying to write to the same special output path. + +**Diagnosis**: + +1. Check if ``group_by`` is being resolved correctly during path planning: + + .. code-block:: python + + # Add debug logging in path_planner.py::_get_execution_groups() + logger.info(f"🔍 PATH_PLANNER: group_by={group_by} (via ObjectState)") + logger.info(f"🔍 PATH_PLANNER: Resolved groups: {result}") + +2. Check if ``special_outputs_by_group`` has the expected groups: + + .. code-block:: python + + # Add debug logging in function_step.py::_process_single_pattern_group() + logger.info(f"🔍 AVAILABLE_GROUPS: {list(special_outputs_by_group.keys())}") + logger.info(f"🔍 COMPONENT_KEY: {component_key}") + +**Common Causes**: + +1. **``group_by`` is ``None`` during path planning**: The path planner is reading ``group_by`` before it's been resolved from lazy dataclass + + **Fix**: Ensure ``step_state_map`` is passed to ``PathPlanner`` and use ``ObjectState.get_saved_resolved_value()`` + +2. **Pattern discovery not grouping patterns**: ``auto_detect_patterns()`` not receiving ``group_by`` parameter + + **Fix**: Ensure ``group_by`` is passed from ``FunctionStep.process()`` to ``microscope_handler.auto_detect_patterns()`` + +3. **Orchestrator not initialized**: Cannot resolve component keys for ``group_by`` + + **Fix**: Ensure orchestrator is initialized before compilation + +Issue 2: ``group_by`` Resolves to ``None`` During Compilation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Symptom**: Path planner logs show ``group_by=None`` even though step configuration has ``group_by=GroupBy.CHANNEL`` + +**Root Cause**: Lazy dataclass not resolved via ObjectState during path planning. + +**Diagnosis**: + +.. code-block:: python + + # Check if using direct getattr (WRONG): + group_by = getattr(step.processing_config, "group_by", None) # Returns unresolved lazy value + + # Should use ObjectState (CORRECT): + group_by = step_state.get_saved_resolved_value("processing_config.group_by") + +**Fix**: Update ``PathPlanner._get_execution_groups()`` to accept ``step_index`` and use ``step_state_map`` for resolution. + +**Key Commit**: The compiler was refactored to resolve step attributes via ObjectState instead of getattr with fallback. + +Issue 3: Understanding "default" vs ``None`` in Group Keys +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Confusion**: Why do some logs show ``"default"`` and others show ``None``? + +**Explanation**: + +- **Internal dict keys**: Use string ``"default"`` for non-dict patterns (see ``func_arg_prep.py::iter_pattern_items()``) +- **Special output group keys**: Convert ``"default"`` to ``None`` (see ``path_planner.py::extract_attributes()`` line 59) + +.. code-block:: python + + # Internal representation: + grouped_patterns = {"default": [pattern1, pattern2]} + + # Special output groups: + special_outputs_by_group = {None: {"cell_counts": {"path": "..."}}} + + # Conversion happens here: + normalized_key = None if group_key == "default" else group_key + +**When you see**: +- ``dict_key_for_funcplan = "default"``: List/single pattern execution +- ``special_outputs_by_group = {None: ...}``: Ungrouped special outputs +- ``component_key = None``: No component grouping + +Variable Components vs Group By +-------------------------------- + +Understanding the Difference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**``variable_components``**: Controls which components vary during pattern discovery + +.. code-block:: python + + variable_components = [VariableComponents.SITE, VariableComponents.CHANNEL] + # Discovers patterns where site and channel vary: + # "A01_s{iii}_w1_z001.tif" ← site varies, channel=1 fixed + # "A01_s{iii}_w2_z001.tif" ← site varies, channel=2 fixed + +**``group_by``**: Controls how discovered patterns are organized and how special outputs are namespaced + +.. code-block:: python + + group_by = GroupBy.CHANNEL + # Groups patterns by channel value: + # {"1": ["A01_s{iii}_w1_z001.tif"], "2": ["A01_s{iii}_w2_z001.tif"]} + # Creates channel-specific special output paths + +**Key Distinction**: +- ``variable_components``: "What varies in the pattern?" (pattern discovery) +- ``group_by``: "How should we organize the patterns?" (grouping and namespacing) + +Example Combinations +~~~~~~~~~~~~~~~~~~~~~ + +**Example 1: Site varies, no grouping** + +.. code-block:: python + + FunctionStep( + func=(normalize, {}), + variable_components=[VariableComponents.SITE], + group_by=None + ) + # Discovers: ["A01_s{iii}_w1_z001.tif"] + # Groups: {"default": ["A01_s{iii}_w1_z001.tif"]} + # Special outputs: All sites write to same path + +**Example 2: Site and channel vary, group by channel** + +.. code-block:: python + + FunctionStep( + func=(normalize, {}), + variable_components=[VariableComponents.SITE, VariableComponents.CHANNEL], + group_by=GroupBy.CHANNEL + ) + # Discovers: ["A01_s{iii}_w1_z001.tif", "A01_s{iii}_w2_z001.tif"] + # Groups: {"1": ["A01_s{iii}_w1_z001.tif"], "2": ["A01_s{iii}_w2_z001.tif"]} + # Special outputs: Each channel gets its own path + +**Example 3: Dict pattern (group_by specifies key meaning)** + +.. code-block:: python + + FunctionStep( + func={'1': analyze_nuclei, '2': analyze_gfp}, + variable_components=[VariableComponents.SITE], + group_by=GroupBy.CHANNEL # Keys '1' and '2' are channel numbers + ) + # Discovers: ["A01_s{iii}_w1_z001.tif", "A01_s{iii}_w2_z001.tif"] + # Groups: {"1": ["A01_s{iii}_w1_z001.tif"], "2": ["A01_s{iii}_w2_z001.tif"]} + # Routes: channel 1 → analyze_nuclei, channel 2 → analyze_gfp + + diff --git a/docs/source/architecture/pipeline_compilation_system.rst b/docs/source/architecture/pipeline_compilation_system.rst index fa673f3cc..de7035014 100644 --- a/docs/source/architecture/pipeline_compilation_system.rst +++ b/docs/source/architecture/pipeline_compilation_system.rst @@ -45,19 +45,28 @@ upon the previous: Phase 1: Step Plan Initialization (``initialize_step_plans_for_context``) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -**Purpose**: Establishes the data flow topology and initializes step plans +**Purpose**: Establishes the data flow topology, registers ObjectState layers, +and resolves a deterministic, saved-value representation of each step - **Input**: Step definitions + ProcessingContext (well_id, input_dir) - **Output**: Initialized ``step_plans`` with input/output directories and - special I/O paths -- **Responsibilities**: - - - Creates basic step plan structure for each step - - Calls ``PipelinePathPlanner.prepare_pipeline_paths()`` for path resolution - - Determines input/output directories for each step - - Creates VFS paths for special I/O (cross-step communication) - - Links special outputs from one step to special inputs of another - - Handles chain breaker logic and input source detection + special I/O paths + - **Responsibilities**: + + - Creates basic step plan structure for each step + - Calls ``PipelinePathPlanner.prepare_pipeline_paths()`` for path resolution + - Registers global/orchestrator/step ``ObjectState`` scopes and resolves + their saved values before metadata injection + - Determines input/output directories for each step + - Creates VFS paths for special I/O (cross-step communication) + - Links special outputs from one step to special inputs of another + - Handles chain breaker logic and input source detection + +**Key Insight**: +The compiler returns ``(resolved_steps, step_state_map)``, and the +``step_state_map`` is consumed by later phases (notably memory +validation/streaming config collection) so only the saved configuration +values used during the resolution pass influence execution plans. **Key Error**: ``"Context step_plans must be initialized before path planning"`` - @@ -315,21 +324,32 @@ ProcessingContext Lifecycle .. code:: python - # Phase 1: Step plan initialization - PipelineCompiler.initialize_step_plans_for_context(context, steps, orchestrator) - - # Phase 2: ZARR store declaration - PipelineCompiler.declare_zarr_stores_for_context(context, steps, orchestrator) - - # Phase 3: Materialization planning - PipelineCompiler.plan_materialization_flags_for_context(context, steps, orchestrator) - - # Phase 4: Memory contract validation + function pattern storage - PipelineCompiler.validate_memory_contracts_for_context(context, steps, orchestrator) - # This phase validates memory types AND stores function patterns in step_plans['func'] - - # Phase 5: GPU resource assignment - PipelineCompiler.assign_gpu_resources_for_context(context, steps, orchestrator) + # Phase 1: Step plan initialization + resolved_steps, step_state_map = PipelineCompiler.initialize_step_plans_for_context( + context, + steps, + orchestrator, + steps_already_resolved=False, + ) + + # Phase 2: ZARR store declaration + PipelineCompiler.declare_zarr_stores_for_context(context, resolved_steps, orchestrator) + + # Phase 3: Materialization planning + PipelineCompiler.plan_materialization_flags_for_context(context, resolved_steps, orchestrator) + + # Phase 4: Memory contract validation + function pattern storage + PipelineCompiler.validate_memory_contracts_for_context( + context, + resolved_steps, + orchestrator, + step_state_map=step_state_map, + ) + # This phase validates memory types, stores function patterns in step_plans['func'], + # and builds ``context.required_visualizers`` using streaming configs from the ObjectState map. + + # Phase 5: GPU resource assignment + PipelineCompiler.assign_gpu_resources_for_context(context, resolved_steps, orchestrator) 3. Freezing ~~~~~~~~~~~ diff --git a/docs/source/architecture/plate_manager_services.rst b/docs/source/architecture/plate_manager_services.rst index f03bde8e7..82ff4e3c5 100644 --- a/docs/source/architecture/plate_manager_services.rst +++ b/docs/source/architecture/plate_manager_services.rst @@ -1,213 +1,63 @@ PlateManager Services Architecture -=================================== +================================== -The Problem: Widget-Embedded Business Logic --------------------------------------------- +The Problem: Parallel Service Paths Drifted +------------------------------------------- -The PlateManager widget originally contained all business logic: orchestrator initialization, pipeline compilation, ZMQ client lifecycle management, and execution polling. This made the widget hard to test (requires PyQt setup), hard to reuse (logic is tied to Qt), and hard to debug (business logic mixed with UI concerns). +The previous PlateManager architecture split orchestration across independent +``CompilationService`` and ``ZMQExecutionService`` modules. That split caused +state duplication and stale UI projections because compile and execute flows +could diverge in how they updated plate state, progress, and status strings. -The Solution: Protocol-Based Service Extraction ------------------------------------------------- - -The PlateManager delegates business logic to two protocol-based services: CompilationService and ZMQExecutionService. These services implement clean interfaces (Protocols) that define what the widget needs without coupling to Qt. This enables testing services independently, reusing them in other contexts, and understanding business logic without Qt knowledge. - -Overview --------- +Current Service Model +--------------------- -The PlateManager widget delegates business logic to two protocol-based services: +PlateManager now uses a unified service boundary: -- ``CompilationService`` (~205 lines): Orchestrator initialization and pipeline compilation -- ``ZMQExecutionService`` (~305 lines): ZMQ client lifecycle and execution polling +- ``BatchWorkflowService`` for compile + execute orchestration +- ``ExecutionServerStatusPresenter`` for consistent status-line rendering +- ``progress_batch_reset`` for deterministic batch boundary resets +- ``plate_config_resolver`` for canonical per-plate pipeline config lookup -This service extraction reduces widget complexity by separating concerns and enables -testability through protocol-based interfaces. +This service layer keeps the widget focused on UI state and delegates workflow +policy to reusable service classes. -CompilationService +Canonical Workflow ------------------ -**Location**: ``openhcs/pyqt_gui/widgets/shared/services/compilation_service.py`` - -**Purpose**: Manages orchestrator initialization and pipeline compilation. - -**Protocol Interface**: - -.. code-block:: python - - from typing import Protocol - - class CompilationHost(Protocol): - """Protocol for widgets that host compilation operations.""" - - def get_global_config(self) -> GlobalPipelineConfig: - """Get global pipeline configuration.""" - ... - - def get_pipeline_data(self) -> Dict[str, List[AbstractStep]]: - """Get pipeline data (plate_path -> steps).""" - ... - - def set_orchestrator(self, orchestrator: PipelineOrchestrator) -> None: - """Set the orchestrator instance.""" - ... - - def on_compilation_success(self) -> None: - """Called when compilation succeeds.""" - ... - - def on_compilation_error(self, error: str) -> None: - """Called when compilation fails.""" - ... - -**Usage**: - -.. code-block:: python - - from openhcs.pyqt_gui.widgets.shared.services.compilation_service import ( - CompilationService, - CompilationHost - ) - - class PlateManagerWidget(AbstractManagerWidget, CompilationHost): - def __init__(self): - super().__init__() - self.compilation_service = CompilationService() - - def action_compile(self): - """Compile button handler.""" - self.compilation_service.compile_plates(self) - - # Implement CompilationHost protocol - def get_global_config(self) -> GlobalPipelineConfig: - return self.global_config - - def get_pipeline_data(self) -> Dict[str, List[AbstractStep]]: - return self.pipeline_data - - def set_orchestrator(self, orchestrator: PipelineOrchestrator) -> None: - self.orchestrator = orchestrator - - def on_compilation_success(self) -> None: - self.status_label.setText("Compilation successful") - - def on_compilation_error(self, error: str) -> None: - QMessageBox.critical(self, "Compilation Error", error) - -**Key Methods**: - -- ``compile_plates(host: CompilationHost)``: Main compilation entry point -- ``_validate_pipeline_steps(pipeline_data)``: Validate pipeline structure -- ``_get_or_create_orchestrator(host)``: Initialize or reuse orchestrator - -ZMQExecutionService -------------------- - -**Location**: ``openhcs/pyqt_gui/widgets/shared/services/zmq_execution_service.py`` - -**Purpose**: Manages ZMQ client lifecycle, execution polling, and progress updates. - -**Protocol Interface**: - -.. code-block:: python - - from typing import Protocol - - class ExecutionHost(Protocol): - """Protocol for widgets that host execution operations.""" - - def get_orchestrator(self) -> Optional[PipelineOrchestrator]: - """Get the orchestrator instance.""" - ... - - def on_execution_started(self) -> None: - """Called when execution starts.""" - ... - - def on_execution_progress(self, progress: float, status: str) -> None: - """Called on progress updates.""" - ... - - def on_execution_complete(self) -> None: - """Called when execution completes.""" - ... - - def on_execution_error(self, error: str) -> None: - """Called when execution fails.""" - ... - -**Usage**: - -.. code-block:: python - - from openhcs.pyqt_gui.widgets.shared.services.zmq_execution_service import ( - ZMQExecutionService, - ExecutionHost - ) - - class PlateManagerWidget(AbstractManagerWidget, ExecutionHost): - def __init__(self): - super().__init__() - self.execution_service = ZMQExecutionService() - - def action_run(self): - """Run button handler.""" - self.execution_service.run_plates(self) - - def action_stop(self): - """Stop button handler.""" - self.execution_service.stop_execution(self) - - # Implement ExecutionHost protocol - def get_orchestrator(self) -> Optional[PipelineOrchestrator]: - return self.orchestrator - - def on_execution_started(self) -> None: - self.run_button.setEnabled(False) - self.stop_button.setEnabled(True) - - def on_execution_progress(self, progress: float, status: str) -> None: - self.progress_bar.setValue(int(progress * 100)) - self.status_label.setText(status) - - def on_execution_complete(self) -> None: - self.run_button.setEnabled(True) - self.stop_button.setEnabled(False) - QMessageBox.information(self, "Success", "Execution complete") - - def on_execution_error(self, error: str) -> None: - QMessageBox.critical(self, "Execution Error", error) - -**Key Methods**: - -- ``run_plates(host: ExecutionHost)``: Start execution with ZMQ polling -- ``stop_execution(host: ExecutionHost)``: Stop execution and cleanup -- ``disconnect(host: ExecutionHost)``: Disconnect ZMQ client -- ``_poll_execution_status(host)``: Poll for progress updates (runs in timer) - -Architecture Benefits ---------------------- +Compile-only flow: -**Separation of Concerns**: +1. Build ``CompileJob`` objects from selected plates. +2. Submit all compile jobs through the workflow service compile backend. +3. Wait all compile jobs. +4. Store compiled artifacts and mark orchestrator states. -- Widget focuses on UI state and user interactions -- Services handle business logic and external communication -- Clear boundaries via protocol interfaces +Run flow: -**Testability**: +1. Reset progress for a new batch. +2. Compile all selected plates first. +3. Submit all execution jobs with compile artifact IDs. +4. Poll completion per execution and converge states in one place. -- Services can be tested independently with mock hosts -- Protocol interfaces enable dependency injection -- No tight coupling to specific widget implementations +Key Invariants +-------------- -**Reusability**: +- One service owns compile + run orchestration. +- Batch reset always clears old progress before new work starts. +- Plate list state is updated from a single source (workflow service callbacks). +- Progress projection and server status text are derived data, not mutable cache. -- Services can be used by any widget implementing the protocol -- Consistent behavior across different UI contexts -- Easy to add new hosts (e.g., TUI, CLI) +Primary Modules +--------------- -See Also --------- +- ``openhcs/pyqt_gui/widgets/shared/services/batch_workflow_service.py`` +- ``openhcs/pyqt_gui/widgets/shared/services/execution_server_status_presenter.py`` +- ``openhcs/pyqt_gui/widgets/shared/services/progress_batch_reset.py`` +- ``openhcs/pyqt_gui/widgets/shared/services/plate_config_resolver.py`` -- :doc:`abstract_manager_widget` - ABC for manager widgets -- :doc:`ui_services_architecture` - ParameterFormManager services -- :doc:`gui_performance_patterns` - Cross-window preview system +Related Architecture Pages +-------------------------- +- :doc:`batch_workflow_service` +- :doc:`progress_runtime_projection_system` +- :doc:`zmq_server_browser_system` diff --git a/docs/source/architecture/progress_runtime_projection_system.rst b/docs/source/architecture/progress_runtime_projection_system.rst new file mode 100644 index 000000000..7da5748ff --- /dev/null +++ b/docs/source/architecture/progress_runtime_projection_system.rst @@ -0,0 +1,49 @@ +Progress Runtime Projection System +================================== + +Scope +----- + +This page documents the OpenHCS implementation path only. + +The generic progress abstraction lives in ``zmqruntime.progress`` and is +documented in the ``zmqruntime`` docs. + +OpenHCS Implementation Path +--------------------------- + +OpenHCS-specific modules: + +- ``openhcs.core.progress.types`` (domain event vocabulary + semantics) +- ``openhcs.core.progress.registry`` (OpenHCS semantic keying wrapper) +- ``openhcs.core.progress.projection`` (OpenHCS adapter + runtime projection API) +- ``openhcs.core.progress.emitters`` (orchestrator/step event emission) + +Consumption path: + +1. Emit typed ``ProgressEvent`` values from compile/execute flows. +2. Route events through explicit queue wiring via ``set_progress_queue(...)``. +3. Register events in ``ProgressRegistry`` with semantic channel keying. +4. Build runtime projection for UI status derivation. +5. Render projection in plate manager and ZMQ server browser services. + +OpenHCS Invariants +------------------ + +- Canonical execution tree path is ``plate -> worker -> well -> step``. +- ``PATTERN_GROUP`` is step-detail only and does not set well pipeline percent. +- Plate/server status is derived from projection snapshots, not mutable cache. + +Canonical Abstraction Docs +-------------------------- + +See ``zmqruntime`` for abstraction internals: + +- ``external/zmqruntime/docs/source/architecture/progress_registry_projection.rst`` +- ``external/zmqruntime/docs/source/architecture/zmq_execution_system.rst`` + +Related OpenHCS Pages +--------------------- + +- :doc:`batch_workflow_service` +- :doc:`zmq_server_browser_system` diff --git a/docs/source/architecture/roi_system.rst b/docs/source/architecture/roi_system.rst index 6474442c0..f18d4db3b 100644 --- a/docs/source/architecture/roi_system.rst +++ b/docs/source/architecture/roi_system.rst @@ -362,22 +362,24 @@ Streams ROIs as ImageJ ROIs to ROI Manager via ZMQ: rm.addRoi(java_roi) Integration with Pipeline System -================================= +================================ Special Outputs Registration ----------------------------- -Functions register ROI outputs using the ``@special_outputs`` decorator: +Functions register ROI outputs using the ``@special_outputs`` decorator with writer-based materialization specs: .. code-block:: python + from openhcs.processing.materialization import MaterializationSpec, CsvOptions, ROIOptions + @special_outputs( - cell_counts=materialize_cell_counts, - segmentation_masks=materialize_segmentation_masks + ("cell_counts", MaterializationSpec(CsvOptions(filename_suffix="_details.csv"))), + ("segmentation_masks", MaterializationSpec(ROIOptions())), ) def count_cells_single_channel(..., return_segmentation_mask: bool = False): """Count cells and optionally return segmentation masks.""" - + if return_segmentation_mask: return output_stack, cell_counts, labeled_masks else: @@ -388,7 +390,7 @@ Materialization Workflow 1. **Execution**: Function returns special outputs (e.g., labeled masks) 2. **Storage**: Special outputs saved to memory backend with channel-specific paths -3. **Materialization**: After step completion, materialization handlers are invoked +3. **Materialization**: After step completion, format writers are invoked 4. **Multi-Backend**: ROIs saved to all backends (disk + streaming) simultaneously **Path Transformation**: diff --git a/docs/source/architecture/scope_window_factory_system.rst b/docs/source/architecture/scope_window_factory_system.rst new file mode 100644 index 000000000..633766054 --- /dev/null +++ b/docs/source/architecture/scope_window_factory_system.rst @@ -0,0 +1,361 @@ +Scope Window Factory System +=========================== + +Overview +-------- + +The scope window factory system provides a pattern-based handler mechanism for creating windows based on ``scope_id`` values. This system replaces monolithic factory classes with a flexible registration pattern that allows adding new window types without modifying core code. + +**Module**: ``pyqt_reactive.services.scope_window_factory`` + +Core Components +--------------- + +ScopeWindowRegistry +~~~~~~~~~~~~~~~~~~ + +Central registry that maps regex patterns to handler functions. + +.. code-block:: python + + from pyqt_reactive.services.scope_window_factory import ScopeWindowRegistry + + # Register handlers at application startup + ScopeWindowRegistry.register_handler( + pattern=r"^/path/to/plate$", + handler=create_plate_config_window + ) + +Scope ID Pattern Matching +~~~~~~~~~~~~~~~~~~~~~~~~ + +Window creation is triggered by matching ``scope_id`` strings against registered patterns: + +- **Global config**: ``""`` (empty string) +- **Plate configs**: ``/path/to/plate`` (no ``::`` separator) +- **Plate list root**: ``__plates__`` (special root state) +- **Step editors**: ``/path/to/plate::step_N`` (``::step_N`` suffix) +- **Function scopes**: ``/path/to/plate::step_N::func_M`` (additional ``::func_M``) + +Registration Order +~~~~~~~~~~~~~~~~~ + +Patterns are evaluated in registration order. More specific patterns should be registered first: + +.. code-block:: python + + def register_openhcs_window_handlers(): + # Order matters - more specific patterns first + + # Step/function editors (match ::step_N or ::step_N::func_M) + ScopeWindowRegistry.register_handler( + pattern=r"^.*::step_\d+(::func_\d+)?$", + handler=_create_step_editor_window + ) + + # Plate configs (match /path - no :: separator) + ScopeWindowRegistry.register_handler( + pattern=r"^/[^:]*$", + handler=_create_plate_config_window + ) + + # Plates root list + ScopeWindowRegistry.register_handler( + pattern=r"^__plates__$", + handler=_create_plates_root_window + ) + + # Global config (empty string) + ScopeWindowRegistry.register_handler( + pattern=r"^$", + handler=_create_global_config_window + ) + +Handler Functions +---------------- + +Handler functions create and show windows for a given ``scope_id`` and optional ``object_state``. + +**Signature**: + +.. code-block:: python + + def handler(scope_id: str, object_state=None) -> Optional[QWidget]: + """Create and show a window for the given scope_id. + + Args: + scope_id: The scope identifier (pattern-matched) + object_state: Optional ObjectState instance (for time-travel) + + Returns: + QWidget: The created window, or None if no window should be created + """ + +**Example Handler**: + +.. code-block:: python + + from pyqt_reactive.services.scope_window_factory import ScopeWindowRegistry + + def _create_global_config_window(scope_id: str, object_state=None) -> Optional[QWidget]: + """Create GlobalPipelineConfig editor window.""" + from openhcs.pyqt_gui.windows.config_window import ConfigWindow + from openhcs.core.config import GlobalPipelineConfig + from openhcs.config_framework.global_config import ( + get_current_global_config, + set_global_config_for_editing, + ) + + current_config = ( + get_current_global_config(GlobalPipelineConfig) or GlobalPipelineConfig() + ) + + def handle_save(new_config): + set_global_config_for_editing(GlobalPipelineConfig, new_config) + + window = ConfigWindow( + config_class=GlobalPipelineConfig, + current_config=current_config, + on_save_callback=handle_save, + scope_id=scope_id, + ) + window.show() + window.raise_() + window.activateWindow() + return window + +Window Creation +--------------- + +Via WindowFactory +~~~~~~~~~~~~~~~~~ + +Use the generic ``WindowFactory`` class to create windows: + +.. code-block:: python + + from pyqt_reactive.services import WindowFactory + + # Create window for a scope + window = WindowFactory.create_window_for_scope(scope_id, object_state) + +Via WindowManager (with Time-Travel) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For windows that should be managed as singletons: + +.. code-block:: python + + from openhcs.pyqt_gui.services.window_manager import WindowManager + + def show_config_window(): + def factory(): + return ConfigWindow(...) + + window = WindowManager.show_or_focus(scope_id, factory) + +Time-Travel Integration +----------------------- + +The scope window factory integrates with time-travel through the ``object_state`` parameter: + +1. **Time-Travel Reopens Windows**: When time-traveling to a dirty state, the system: + - Calls ``WindowFactory.create_window_for_scope(scope_id, object_state)`` + - Handler receives the dirty ``object_state`` for proper reconstruction + - Window is shown and focused + +2. **ObjectState for Context**: Handlers use ``object_state`` to reconstruct UI state: + +.. code-block:: python + + def _create_step_editor_window(scope_id: str, object_state=None): + """Create step editor with time-travel support.""" + # Get step from object_state (if provided) + if object_state: + step = object_state.object_instance + else: + # Find step by token (provenance navigation) + step = _find_step_by_token(plate_path, step_token) + + window = DualEditorWindow( + step_data=step, + orchestrator=orchestrator, + scope_id=scope_id, + ) + window.show() + return window + +Registration in OpenHCS +------------------------ + +Handlers are registered during application initialization in ``main.py``: + +.. code-block:: python + + from openhcs.pyqt_gui.services.window_handlers import ( + register_openhcs_window_handlers, + ) + + class OpenHCSMainWindow(QMainWindow): + def __init__(self): + super().__init__() + # ... other initialization ... + + # Register OpenHCS window handlers + register_openhcs_window_handlers() + +Handler Implementation Guide +-------------------------- + +Plate Config Handler +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + def _create_plate_config_window(scope_id: str, object_state=None): + from openhcs.pyqt_gui.windows.config_window import ConfigWindow + from openhcs.core.config import PipelineConfig + from pyqt_reactive.services import ServiceRegistry + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + + # Get plate manager from ServiceRegistry + plate_manager = ServiceRegistry.get(PlateManagerWidget) + if not plate_manager: + return None + + orchestrator = ObjectStateRegistry.get_object(scope_id) + if not orchestrator: + return None + + window = ConfigWindow( + config_class=PipelineConfig, + current_config=orchestrator.pipeline_config, + on_save_callback=None, # ObjectState handles save + scope_id=scope_id, + ) + window.show() + window.raise_() + window.activateWindow() + return window + +Step Editor Handler +~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + def _create_step_editor_window(scope_id: str, object_state=None): + from openhcs.pyqt_gui.windows.dual_editor_window import DualEditorWindow + + parts = scope_id.split("::") + if len(parts) < 2: + return None + + plate_path = parts[0] + step_token = parts[1] + is_function_scope = len(parts) >= 3 + + orchestrator = ObjectStateRegistry.get_object(plate_path) + if not orchestrator: + return None + + # Get step from object_state (time-travel) or find by token + step = None + if object_state: + step_state = ObjectStateRegistry.get_by_scope(step_scope_id) + step = step_state.object_instance if step_state else None + else: + step = _find_step_by_token(plate_manager, plate_path, step_token) + + if not step: + return None + + window = DualEditorWindow( + step_data=step, + is_new=False, + on_save_callback=None, + orchestrator=orchestrator, + parent=None, + ) + + if is_function_scope and window.tab_widget: + window.tab_widget.setCurrentIndex(1) + + window.show() + window.raise_() + window.activateWindow() + return window + +Special Case: No Window +~~~~~~~~~~~~~~~~~~~~~~~ + +Some scopes (like ``__plates__``) represent state without a window: + +.. code-block:: python + + def _create_plates_root_window(scope_id: str, object_state=None): + """Root plate list state - no window to create.""" + logger.debug(f"[WINDOW_FACTORY] Skipping window creation for __plates__ scope") + return None + +Best Practices +-------------- + +Pattern Specificity +~~~~~~~~~~~~~~~~~~~ + +- **Specific first**: Register patterns with ``::`` separators before generic paths +- **Global last**: Register ``^$`` (empty string) handler last +- **Test order**: Verify patterns match expected scopes in correct order + +Error Handling +~~~~~~~~~~~~~~ + +- Return ``None`` when window cannot be created +- Log warnings for missing dependencies +- Validate scope_id format before processing + +Window Display +~~~~~~~~~~~~~~ + +Always call these three methods after creating a window: + +.. code-block:: python + + window.show() + window.raise_() + window.activateWindow() + +Scope ID Design +~~~~~~~~~~~~~~~~ + +- Use filesystem-like paths: ``/path/to/plate`` +- Use ``::`` for hierarchy: ``plate::step_N::func_M`` +- Keep identifiers stable (don't change scope IDs for existing data) + +Integration Points +------------------ + +WindowManager Integration +~~~~~~~~~~~~~~~~~~~~~~~~ + +``WindowManager`` calls ``WindowFactory.create_window_for_scope()`` internally: + +.. code-block:: python + + # WindowManager uses scope window factory for time-travel + window = WindowFactory.create_window_for_scope(scope_id, state) + +Provenance Navigation +~~~~~~~~~~~~~~~~~~~~~~ + +Scope IDs appear in provenance tracking. Clicking a provenance entry: +1. Calls ``WindowManager.focus_and_navigate(scope_id)`` +2. If window doesn't exist, calls ``WindowFactory.create_window_for_scope()`` +3. Handler creates window and navigates to the field + +See Also +-------- + +- :doc:`/architecture/service_registry_integration` - ServiceRegistry pattern +- :doc:`/development/window_manager_usage` - Window management with singletons +- :doc:`/architecture/time_travel_system` - Time-travel architecture diff --git a/docs/source/architecture/service_registry_integration.rst b/docs/source/architecture/service_registry_integration.rst new file mode 100644 index 000000000..666f8b342 --- /dev/null +++ b/docs/source/architecture/service_registry_integration.rst @@ -0,0 +1,466 @@ +Service Registry Integration +========================== + +Overview +-------- + +The ServiceRegistry provides centralized service and widget management, eliminating circular dependencies and simplifying component discovery. Widgets and services register themselves at creation time, making them available throughout the application without manual tracking. + +**Module**: ``pyqt_reactive.services.service_registry`` + +Core Concepts +------------- + +Service Registration +~~~~~~~~~~~~~~~~~~ + +Services register themselves via the ``AutoRegisterServiceMixin``: + +.. code-block:: python + + from pyqt_reactive.services import ServiceRegistry, AutoRegisterServiceMixin + + class PlateManagerWidget(QWidget, AutoRegisterServiceMixin): + """Plate manager widget auto-registers with ServiceRegistry.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # AutoRegisterServiceMixin registers this instance automatically + +Service Resolution +~~~~~~~~~~~~~~~~~~~ + +Services are retrieved by type: + +.. code-block:: python + + from pyqt_reactive.services import ServiceRegistry + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + + # Get plate manager widget + plate_manager = ServiceRegistry.get(PlateManagerWidget) + + if plate_manager: + # Use the widget + plate_manager.refresh_plate_list() + +AutoRegisterServiceMixin +------------------------ + +Mixin for automatic service registration. + +**Usage**: + +.. code-block:: python + + class MyWidget(QWidget, AutoRegisterServiceMixin): + """Widget that auto-registers with ServiceRegistry.""" + + def __init__(self): + super().__init__() + # ServiceRegistry.set(MyWidget, self) called automatically + +**Behind the scenes**: + +.. code-block:: python + + class AutoRegisterServiceMixin: + """Mixin to auto-register widgets with ServiceRegistry.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + ServiceRegistry.set(type(self), self) + +ServiceRegistry API +------------------- + +Register Services +~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from pyqt_reactive.services import ServiceRegistry + + # Register a service instance + ServiceRegistry.set(MyService, my_service_instance) + + # Register with explicit service key + ServiceRegistry.set("my_service_key", my_service_instance) + +Retrieve Services +~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + # Get by type (preferred) + plate_manager = ServiceRegistry.get(PlateManagerWidget) + + # Get by key + service = ServiceRegistry.get("my_service_key") + + # Get with default fallback + manager = ServiceRegistry.get(PlateManagerWidget, None) + +Check Registration +~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + # Check if service exists + if ServiceRegistry.has(PlateManagerWidget): + # Use it + pass + + # Check with key + if ServiceRegistry.has("my_service"): + pass + +Clear Services +~~~~~~~~~~~~~~ + +.. code-block:: python + + # Clear a specific service + ServiceRegistry.clear(MyService) + + # Clear all services (use with caution) + ServiceRegistry.clear_all() + +Common Use Cases +---------------- + +Widget-to-Widget Communication +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Previously required ``floating_windows`` dictionary or ``QApplication`` traversal: + +.. code-block:: python + + # Before: Traversal through all windows + for widget in QApplication.topLevelWidgets(): + if hasattr(widget, 'floating_windows'): + plate_dialog = widget.floating_windows.get("plate_manager") + if plate_dialog: + plate_manager = plate_dialog.findChild(PlateManagerWidget) + break + +Now use ServiceRegistry: + +.. code-block:: python + + # After: Direct lookup + from pyqt_reactive.services import ServiceRegistry + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + + plate_manager = ServiceRegistry.get(PlateManagerWidget) + if plate_manager: + # Connect signals + self.plate_selected.connect(plate_manager.set_current_plate) + +Window Handler Registration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Handlers access widgets from ServiceRegistry: + +.. code-block:: python + + def _create_plate_config_window(scope_id: str, object_state=None): + from openhcs.pyqt_gui.windows.config_window import ConfigWindow + from pyqt_reactive.services import ServiceRegistry + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + + # Get plate manager from ServiceRegistry + plate_manager = ServiceRegistry.get(PlateManagerWidget) + if not plate_manager: + logger.warning("Could not find PlateManager for plate config window") + return None + + orchestrator = ObjectStateRegistry.get_object(scope_id) + if not orchestrator: + return None + + window = ConfigWindow( + config_class=PipelineConfig, + current_config=orchestrator.pipeline_config, + scope_id=scope_id, + ) + window.show() + return window + +Pipeline-to-Plate Manager Connection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Connect pipeline editor to plate manager via ServiceRegistry: + +.. code-block:: python + + def _connect_pipeline_to_plate_manager(self, pipeline_widget): + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + from pyqt_reactive.services import ServiceRegistry + + # Get plate manager from ServiceRegistry + plate_manager_widget = ServiceRegistry.get(PlateManagerWidget) + + if plate_manager_widget: + # Connect plate selection signal to pipeline editor + plate_manager_widget.plate_selected.connect( + pipeline_widget.set_current_plate + ) + + # Set current plate if already selected + if plate_manager_widget.selected_plate_path: + pipeline_widget.set_current_plate( + plate_manager_widget.selected_plate_path + ) + + logger.debug("Connected pipeline editor to plate manager") + else: + logger.warning("Could not find plate manager widget to connect") + +Service Lifecycle +----------------- + +Registration Timing +~~~~~~~~~~~~~~~~~~ + +Services are registered at widget creation time: + +.. code-block:: python + + # In main.py + self.plate_manager_widget = PlateManagerWidget(...) + # PlateManagerWidget.__init__ calls ServiceRegistry.set(PlateManagerWidget, self) + + # Plate manager is now available immediately + other_widget.connect_to_plate_manager() + +Unregistration +~~~~~~~~~~~~~~ + +Widgets are unregistered automatically when destroyed (if subclassing QObject): + +.. code-block:: python + + # ServiceRegistry hooks into object destruction + def on_object_destroyed(self): + service_type = self._registered_service_type + ServiceRegistry.clear(service_type) + +Singleton Pattern +~~~~~~~~~~~~~~~~~~ + +ServiceRegistry enforces one instance per service type: + +.. code-block:: python + + # First registration + service1 = MyService() + ServiceRegistry.set(MyService, service1) + + # Second registration overwrites first + service2 = MyService() + ServiceRegistry.set(MyService, service2) # Replaces service1 + + # Only service2 is available + retrieved = ServiceRegistry.get(MyService) + assert retrieved is service2 # True + +Thread Safety +------------- + +ServiceRegistry is **not thread-safe** by default. All service operations should occur on the main GUI thread: + +.. code-block:: python + + # CORRECT: Main thread + def main_thread_function(): + service = ServiceRegistry.get(MyService) + service.do_something() + + # INCORRECT: Background thread + def background_thread_function(): + # This can cause race conditions + service = ServiceRegistry.get(MyService) + service.do_something() + +If thread-safe access is needed, implement a wrapper: + +.. code-block:: python + + from threading import Lock + + class ThreadSafeServiceRegistry: + def __init__(self): + self._services = {} + self._lock = Lock() + + def get(self, service_type, default=None): + with self._lock: + return self._services.get(service_type, default) + + def set(self, service_type, instance): + with self._lock: + self._services[service_type] = instance + +Best Practices +-------------- + +Use Type-Based Keys +~~~~~~~~~~~~~~~~~~~ + +Prefer class types over string keys: + +.. code-block:: python + + # PREFERRED: Type-based + ServiceRegistry.set(PlateManagerWidget, widget) + manager = ServiceRegistry.get(PlateManagerWidget) + + # AVOID: String-based (unless necessary) + ServiceRegistry.set("plate_manager", widget) + manager = ServiceRegistry.get("plate_manager") + +Check Before Use +~~~~~~~~~~~~~~~~ + +Always check if service exists: + +.. code-block:: python + + manager = ServiceRegistry.get(PlateManagerWidget) + if manager: + manager.refresh() + else: + logger.warning("PlateManager not available") + +Auto-Register Widgets +~~~~~~~~~~~~~~~~~~~~~ + +Use ``AutoRegisterServiceMixin`` for widgets: + +.. code-block:: python + + # DO: Auto-register + class PlateManagerWidget(QWidget, AutoRegisterServiceMixin): + def __init__(self): + super().__init__() + # Registered automatically + + # AVOID: Manual registration + class PlateManagerWidget(QWidget): + def __init__(self): + super().__init__() + ServiceRegistry.set(PlateManagerWidget, self) # Boilerplate + +Avoid Circular Dependencies +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +ServiceRegistry breaks dependency chains: + +.. code-block:: python + + # BEFORE: Circular dependency + # PipelineEditor needs PlateManager + # PlateManager needs PipelineEditor + # Both import each other → circular + + # AFTER: ServiceRegistry breaks the cycle + # PipelineEditor imports PlateManagerWidget (type only) + # PlateManagerWidget imports PipelineEditorWidget (type only) + # Both resolve via ServiceRegistry.get() at runtime + +Service Keys Design +~~~~~~~~~~~~~~~~~~ + +Design service keys with clear semantics: + +.. code-block:: python + + # Use concrete widget/service types + ServiceRegistry.set(PlateManagerWidget, widget) + + # For multiple instances, use distinct service classes + class MainPlateManagerWidget(PlateManagerWidget): pass + class SecondaryPlateManagerWidget(PlateManagerWidget): pass + + ServiceRegistry.set(MainPlateManagerWidget, widget1) + ServiceRegistry.set(SecondaryPlateManagerWidget, widget2) + +Migration from floating_windows +------------------------------ + +Before +~~~~~~~ + +.. code-block:: python + + # Main window tracks widgets manually + class OpenHCSMainWindow(QMainWindow): + def __init__(self): + self.floating_windows = {} + + def create_plate_manager(self): + window = PlateManagerWindow() + self.floating_windows["plate_manager"] = window + + def get_plate_manager(self): + if "plate_manager" in self.floating_windows: + window = self.floating_windows["plate_manager"] + return window.findChild(PlateManagerWidget) + return None + +After +~~~~~~ + +.. code-block:: python + + # Widgets auto-register + class PlateManagerWidget(QWidget, AutoRegisterServiceMixin): + pass + + # Any code can access widgets + from pyqt_reactive.services import ServiceRegistry + + plate_manager = ServiceRegistry.get(PlateManagerWidget) + +Integration Points +------------------ + +Window Handlers +~~~~~~~~~~~~~~ + +Window handlers use ServiceRegistry to access widgets: + +.. code-block:: python + + def _create_step_editor_window(scope_id: str, object_state=None): + plate_manager = ServiceRegistry.get(PlateManagerWidget) + orchestrator = ObjectStateRegistry.get_object(plate_path) + + window = DualEditorWindow( + step_data=step, + orchestrator=orchestrator, + ) + return window + +Main Window +~~~~~~~~~~~~ + +Main window removes widget tracking: + +.. code-block:: python + + class OpenHCSMainWindow(QMainWindow): + def __init__(self): + # Before: self.floating_windows = {} + # After: No tracking needed + + self.plate_manager_widget = PlateManagerWidget() + # Auto-registered, accessible everywhere + +See Also +-------- + +- :doc:`/architecture/scope_window_factory_system` - Handler-based window creation +- :doc:`/development/window_manager_usage` - Window management patterns +- :doc:`/architecture/ui_services_architecture` - Service layer architecture diff --git a/docs/source/architecture/special_io_system.rst b/docs/source/architecture/special_io_system.rst index 4a7788436..513ad9bfc 100644 --- a/docs/source/architecture/special_io_system.rst +++ b/docs/source/architecture/special_io_system.rst @@ -8,6 +8,11 @@ The Special I/O system enables sophisticated data exchange between pipeline step **System Evolution**: Originally designed for simple single-function steps, the system was extended to handle dict patterns (multiple functions per step) through sophisticated namespacing and scope promotion rules, similar to symbol resolution in programming language compilers. +.. seealso:: + + :doc:`pattern_grouping_and_special_outputs` + Comprehensive guide to pattern grouping, special output path resolution, and debugging path collisions + Architectural Evolution ----------------------- @@ -53,14 +58,14 @@ The Special I/O system uses a declarative approach where functions simply declar **Special Inputs**: Functions that need data from previous steps declare their requirements using ``@special_inputs``. The system automatically loads this data from the VFS and provides it as function parameters. -**Materialization Support**: Special outputs can optionally include materialization specs that declaratively select a handler (CSV, JSON, ROI ZIP, etc.) for persistent formats. +**Materialization Support**: Special outputs can optionally include materialization specs that declaratively select one or more *format writers* (CSV/JSON/ROI ZIP/TIFF/TEXT) for persistent formats. .. code:: python # Example: Position generation with materialization spec - from openhcs.processing.materialization import csv_materializer + from openhcs.processing.materialization import MaterializationSpec, CsvOptions - @special_outputs(("positions", csv_materializer(fields=["x", "y"], analysis_type="positions"))) + @special_outputs(("positions", MaterializationSpec(CsvOptions(filename_suffix=".csv")))) def generate_positions(image_stack): positions = calculate_positions(image_stack) return processed_image, positions @@ -70,6 +75,10 @@ The Special I/O system uses a declarative approach where functions simply declar def stitch_images(image_stack, positions): return stitch(image_stack, positions) +Note: Writer dispatch is automatically inferred from the options type. No need to specify handler strings. +CsvOptions auto-extracts fields from dataclasses/dicts. The ``fields`` parameter is only needed when you +want to control column ordering or select a subset. + Decorator Implementation ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -77,9 +86,9 @@ The decorators work by attaching metadata to function objects that the compilati **Metadata Attachment**: The decorators add attributes to functions (``__special_outputs__``, ``__special_inputs__``) that the compiler reads during pipeline analysis. This metadata-driven approach means functions remain normal Python functions that can be tested independently. -**Materialization Integration**: When special outputs include materialization specs, the decorator stores both the output keys (for path planning) and the specs (for handler dispatch) as separate attributes. +**Materialization Integration**: When special outputs include materialization specs, the decorator stores both the output keys (for path planning) and the specs (for writer dispatch) as separate attributes. -**Backward Compatibility**: Registered materializer handlers can still be referenced directly where appropriate, but specs are the canonical representation. +**Greenfield Rule**: Materialization is writer-driven. Do not register custom handler functions; declare writer options in MaterializationSpec. Compilation-Time Path Resolution -------------------------------- diff --git a/docs/source/architecture/storage_and_memory_system.rst b/docs/source/architecture/storage_and_memory_system.rst index df916863f..6f1bc3f6b 100644 --- a/docs/source/architecture/storage_and_memory_system.rst +++ b/docs/source/architecture/storage_and_memory_system.rst @@ -557,7 +557,7 @@ The materialization system bridges the gap between computational processing and Special Output Decoration ~~~~~~~~~~~~~~~~~~~~~~~~~ -Functions declare their side effects using the ``@special_outputs`` decorator, which can optionally specify materialization specs (resolved via handlers) for converting data to persistent formats. +Functions declare their side effects using the ``@special_outputs`` decorator, which can optionally specify materialization specs (resolved via format writers) for converting data to persistent formats. **Basic Special Outputs** (memory backend only): @@ -578,10 +578,10 @@ Functions declare their side effects using the ``@special_outputs`` decorator, w .. code:: python - from openhcs.processing.materialization import csv_materializer, roi_zip_materializer + from openhcs.processing.materialization import MaterializationSpec, CsvOptions, ROIOptions - @special_outputs(("cell_counts", csv_materializer(fields=["slice_index", "cell_count"], analysis_type="cell_counts")), - ("masks", roi_zip_materializer())) + @special_outputs(("cell_counts", MaterializationSpec(CsvOptions(filename_suffix="_details.csv"))), + ("masks", MaterializationSpec(ROIOptions()))) def count_cells_with_materialization(image_stack): """Function with materialized special outputs.""" processed_image, cell_counts, segmentation_masks = analyze_cells(image_stack) @@ -593,137 +593,42 @@ Functions declare their side effects using the ``@special_outputs`` decorator, w .. code:: python - from openhcs.processing.materialization import csv_materializer + from openhcs.processing.materialization import MaterializationSpec, CsvOptions - @special_outputs("debug_info", ("analysis_results", csv_materializer(fields=["metric"], analysis_type="analysis_results"))) + @special_outputs("debug_info", ("analysis_results", MaterializationSpec(CsvOptions(filename_suffix=".csv")))) def analyze_with_mixed_outputs(image_stack): """Function with both memory-only and materialized outputs.""" # debug_info stays in memory, analysis_results gets materialized return processed_image, debug_info, analysis_results -Materialization Handler Implementation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Note: Writer dispatch is automatically inferred from options type. No need to specify handler strings. +The ``fields`` parameter is optional and defaults to None (auto-extract all fields). +Only specify ``fields`` when you need to control column ordering or select a subset. -Materialization handlers follow standard signatures and handle the conversion from Python objects to persistent file formats. They receive data from the memory backend and save it using the FileManager with appropriate backend selection. +Writer-Based Materialization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -**Single Backend Signature** (for simple materialization): +Materialization is implemented as a small set of format writers (CSV/JSON/ROI ZIP/TIFF/TEXT). Call sites never register custom materializers. -.. code:: python - - def materialize_function_name(data: Any, path: str, filemanager, backend: str) -> str: - """ - Convert special output data to persistent storage format. - - Args: - data: The special output data from memory backend - path: Base path for output files (from VFS path planning) - filemanager: FileManager instance for backend-agnostic I/O - backend: Backend name (disk, omero_local, napari_stream, fiji_stream) - - Returns: - str: Path to the primary output file created - """ - # Get backend instance and dispatch to backend-specific implementation - backend_obj = filemanager.get_backend(backend) - return backend_obj._save_data(data, path) - -**Multi-Backend Signature** (for simultaneous materialization to multiple backends): - -.. code:: python - - def materialize_function_name( - data: Any, - path: str, - filemanager, - backends: Union[str, List[str]], - backend_kwargs: dict = None - ) -> str: - """ - Convert special output data to multiple backends simultaneously. - - Args: - data: The special output data from memory backend - path: Base path for output files (from VFS path planning) - filemanager: FileManager instance for backend-agnostic I/O - backends: Single backend string or list of backends to save to - backend_kwargs: Dict mapping backend names to their kwargs - - Returns: - str: Path to the primary output file created (first backend) - """ - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - - # Materialize to all backends - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) if backend_kwargs else {} - filemanager.save(data, path, backend, **kwargs) - - return path - -**Real Example - Cell Count Materialization**: +Instead, functions declare what they want persisted by providing writer options to ``MaterializationSpec``. .. code:: python - def materialize_cell_counts(data: List[CellCountResult], path: str, filemanager) -> str: - """Materialize cell counting results as analysis-ready CSV and JSON formats.""" - - # Generate output file paths based on the input path - base_path = path.replace('.pkl', '') - json_path = f"{base_path}.json" - csv_path = f"{base_path}_details.csv" - - # Ensure output directory exists for disk backend - from pathlib import Path - from openhcs.constants.constants import Backend - output_dir = Path(json_path).parent - filemanager.ensure_directory(str(output_dir), Backend.DISK.value) - - # Create summary data - summary = { - "analysis_type": "single_channel_cell_counting", - "total_slices": len(data), - "total_cells_detected": sum(result.cell_count for result in data) - } + from openhcs.processing.materialization import MaterializationSpec, CsvOptions, JsonOptions - # Save JSON summary - import json - json_content = json.dumps(summary, indent=2) - filemanager.save(json_content, json_path, Backend.DISK.value) - - # Create detailed CSV - rows = [] - for result in data: - rows.append({ - 'slice_index': result.slice_index, - 'cell_count': result.cell_count, - 'detection_method': result.detection_method, - 'threshold_value': result.threshold_value - }) - - # Save CSV details - import pandas as pd - df = pd.DataFrame(rows) - csv_content = df.to_csv(index=False) - filemanager.save(csv_content, csv_path, Backend.DISK.value) - - return json_path # Return primary output file - -**Real Example - Position Materialization**: - -.. code:: python + # JSON summary + CSV details, JSON is primary + spec = MaterializationSpec( + JsonOptions(source="summary", filename_suffix=".json"), + CsvOptions(source="details", filename_suffix="_details.csv"), + primary=0, + ) - def materialize_ashlar_positions(data: List[Tuple[float, float]], path: str, filemanager) -> str: - """Materialize tile positions as scientific CSV with grid metadata.""" - csv_path = path.replace('.pkl', '_ashlar_positions.csv') +The framework handles: - # Convert to DataFrame with metadata - import pandas as pd - df = pd.DataFrame(data, columns=['x_position_um', 'y_position_um']) - df['tile_id'] = range(len(df)) +- path normalization (including compound suffix stripping) +- directory creation and overwrite/delete semantics +- multi-backend iteration - # Add grid analysis unique_x = sorted(df['x_position_um'].unique()) unique_y = sorted(df['y_position_um'].unique()) df['grid_dimensions'] = f"{len(unique_y)}x{len(unique_x)}" diff --git a/docs/source/architecture/streaming_boundary_and_wrappers.rst b/docs/source/architecture/streaming_boundary_and_wrappers.rst new file mode 100644 index 000000000..287369659 --- /dev/null +++ b/docs/source/architecture/streaming_boundary_and_wrappers.rst @@ -0,0 +1,39 @@ +Streaming Boundary And Wrappers +=============================== + +Overview +-------- + +OpenHCS now treats streaming as a thin-wrapper integration surface, not as the +owner of generic streaming infrastructure. + +Ownership +--------- + +- ``zmqruntime`` owns transport and execution lifecycle primitives: + ZMQ sockets, control/data channels, ping/pong, execution status contracts. +- ``polystore`` owns streaming payload semantics and receiver-side projection: + streaming data types, component-mode grouping, batching/debounce engines, and + viewer receiver helpers. +- ``openhcs`` owns application wiring: + configuration defaults, orchestrator integration, and OpenHCS-specific viewer + process launch/lifecycle policy. + +Current OpenHCS Responsibility +------------------------------ + +In OpenHCS runtime modules (for example ``openhcs/runtime/fiji_viewer_server.py`` +and ``openhcs/runtime/napari_*``), the server classes should remain thin +wrappers that bind OpenHCS configuration/context into external abstractions. + +They should not re-implement generic projection or batching logic that is +already provided by ``polystore`` receiver-core modules. + +Why This Split +-------------- + +- keeps OpenHCS focused on domain orchestration and UX integration +- improves reuse for other applications that need the same streaming stack +- reduces duplicated logic and inconsistent behavior across viewers +- gives one canonical path per semantic concept + diff --git a/docs/source/architecture/zmq_execution_service_extracted.rst b/docs/source/architecture/zmq_execution_service_extracted.rst index f1631d8c2..65dd65153 100644 --- a/docs/source/architecture/zmq_execution_service_extracted.rst +++ b/docs/source/architecture/zmq_execution_service_extracted.rst @@ -1,209 +1,21 @@ -ZMQ Execution Service (Extracted) -================================= +ZMQ Execution Service (Legacy) +============================== -**ZMQ client lifecycle and plate execution service extracted from PlateManagerWidget.** +Status +------ -*Module: openhcs.pyqt_gui.widgets.shared.services.zmq_execution_service* +``ZMQExecutionService`` has been removed from OpenHCS. -Background: UI-Execution Boundary ---------------------------------- +Reason +------ -Pipeline execution in OpenHCS happens on a ZMQ server—a separate process that runs -pipelines and reports progress. The UI is a client that submits jobs and polls for -status. This separation is essential for long-running microscopy workflows that -can span hours or days. +Execution polling and compile orchestration are no longer split across two +services. OpenHCS now uses a unified workflow service plus typed projection and +status presenter abstractions. -But managing this client-server relationship from a UI widget creates tangled code. -The widget needs to handle connection lifecycle, job submission, progress polling, -graceful vs force shutdown, and state reconciliation when the server restarts. These -concerns have nothing to do with displaying widgets or handling user input. +Replacement +----------- -What This Service Does ----------------------- - -``ZMQExecutionService`` extracts all ZMQ client management from the UI layer. It owns -the ``ZMQExecutionClient`` instance and handles: - -- **Connection lifecycle** - Creating and destroying ZMQ connections -- **Job submission** - Sending compiled orchestrators to the server -- **Progress polling** - Monitoring job status and forwarding updates -- **Shutdown coordination** - Graceful (wait for step) vs force (immediate) termination - -The host widget remains responsible only for displaying status and handling user -actions. When the user clicks "Run", the widget calls ``run_plates()``. When they -click "Stop", it calls ``stop_execution()``. All the complexity of ZMQ communication -is hidden inside the service. - -ExecutionHost Protocol ----------------------- - -Like ``CompilationService``, this service uses a Protocol to define the host interface. -The service needs access to host state (orchestrators, execution IDs) and needs to -call back for status updates: - -.. code-block:: python - - from typing import Protocol - - class ExecutionHost(Protocol): - """Protocol for the widget that hosts ZMQ execution.""" - - # State attributes - execution_state: str - plate_execution_ids: Dict[str, str] - plate_execution_states: Dict[str, str] - orchestrators: Dict[str, Any] - plate_compiled_data: Dict[str, Any] - global_config: Any - current_execution_id: Optional[str] - - # Signal emission methods - def emit_status(self, msg: str) -> None: ... - def emit_error(self, msg: str) -> None: ... - def emit_orchestrator_state(self, plate_path: str, state: str) -> None: ... - def emit_execution_complete(self, result: dict, plate_path: str) -> None: ... - def emit_clear_logs(self) -> None: ... - def update_button_states(self) -> None: ... - def update_item_list(self) -> None: ... - - # Execution completion hooks - def on_plate_completed(self, plate_path: str, status: str, result: dict) -> None: ... - def on_all_plates_completed(self, completed: int, failed: int) -> None: ... - -Using the Service ------------------ - -The pattern mirrors ``CompilationService``. Create the service with a host reference, -then call async methods to trigger execution: - -.. code-block:: python - - from openhcs.pyqt_gui.widgets.shared.services.zmq_execution_service import ( - ZMQExecutionService, ExecutionHost - ) - - class MyWidget(QWidget, ExecutionHost): - def __init__(self): - super().__init__() - self.execution_service = ZMQExecutionService(host=self, port=7777) - - async def run_selected(self): - ready = self._get_ready_plates() - await self.execution_service.run_plates(ready) - -The Three Core Methods -~~~~~~~~~~~~~~~~~~~~~~ - -The service exposes a deliberately minimal API—just three methods that cover all -execution scenarios: - -.. code-block:: python - - async def run_plates(self, ready_items: List[Dict]) -> None: - """Run plates using ZMQ execution client.""" - - async def stop_execution(self, graceful: bool = True) -> None: - """Stop current execution (graceful or force).""" - - async def shutdown(self) -> None: - """Cleanup and disconnect ZMQ client.""" - -Execution Flow --------------- - -When ``run_plates()`` is called, the service orchestrates a complex sequence of -operations. Understanding this flow helps debug execution issues: - -1. **Cleanup** - Disconnect any existing client (prevents resource leaks) -2. **Client Creation** - Create new ``ZMQExecutionClient`` with progress callback -3. **Submission** - Submit each plate's orchestrator to the server -4. **Polling** - Periodically check execution status, invoke callbacks -5. **Completion** - Report final results via host callbacks - -.. code-block:: python - - async def run_plates(self, ready_items: List[Dict]) -> None: - # Cleanup old client - await self._disconnect_client(loop) - - # Create new client - self.zmq_client = ZMQExecutionClient( - port=self.port, - persistent=True, - progress_callback=self._on_progress - ) - - # Submit each plate - for item in ready_items: - orchestrator = self.host.orchestrators[item['path']] - execution_id = await self.zmq_client.submit( - orchestrator=orchestrator, - global_config=self.host.global_config - ) - self.host.plate_execution_ids[item['path']] = execution_id - - # Start polling - await self._poll_until_complete() - -Progress Callbacks ------------------- - -The service provides progress updates via internal callbacks: - -.. code-block:: python - - def _on_progress(self, progress_data: dict) -> None: - """Handle progress update from ZMQ client.""" - plate_path = progress_data.get('plate_path') - status = progress_data.get('status') - - self.host.emit_status(f"Plate {plate_path}: {status}") - - if status == 'completed': - self.host.on_plate_completed(plate_path, status, progress_data) - -Shutdown Handling ------------------ - -.. code-block:: python - - async def stop_execution(self, graceful: bool = True) -> None: - """ - Stop current execution. - - Args: - graceful: If True, wait for current step to complete. - If False, force immediate termination. - """ - if graceful: - await self.zmq_client.request_stop() - else: - await self.zmq_client.force_stop() - - self.host.update_button_states() - -Integration with PlateManager ------------------------------ - -.. code-block:: python - - class PlateManagerWidget(AbstractManagerWidget, ExecutionHost): - def __init__(self): - super().__init__() - self._execution_service = ZMQExecutionService(host=self) - - # ExecutionHost protocol implementation - def on_plate_completed(self, plate_path: str, status: str, result: dict): - self._update_plate_status(plate_path, status) - self.update_item_list() - - async def action_run(self): - ready = self._get_ready_plates() - await self._execution_service.run_plates(ready) - -See Also --------- - -- :doc:`compilation_service` - Compilation service for preparing pipelines -- :doc:`abstract_manager_widget` - ABC that PlateManager inherits from -- :doc:`plate_manager_services` - Other PlateManager service extractions +- :doc:`batch_workflow_service` for compile + execute orchestration +- :doc:`progress_runtime_projection_system` for runtime status projection +- :doc:`zmq_server_browser_system` for tree/browser rendering diff --git a/docs/source/architecture/zmq_server_browser_system.rst b/docs/source/architecture/zmq_server_browser_system.rst new file mode 100644 index 000000000..ca95a8f8c --- /dev/null +++ b/docs/source/architecture/zmq_server_browser_system.rst @@ -0,0 +1,79 @@ +ZMQ Server Browser System +========================= + +Overview +-------- + +OpenHCS implements a thin wrapper over the generic pyqt-reactive browser: + +- Generic base: ``pyqt_reactive.widgets.shared.ZMQServerBrowserWidgetABC`` +- OpenHCS adapter: ``openhcs.pyqt_gui.widgets.shared.zmq_server_manager.ZMQServerManagerWidget`` + +This split keeps UI infrastructure generic while preserving OpenHCS-specific +progress semantics and topology validation. + +Boundary +-------- + +This page documents OpenHCS-owned browser behavior: + +- progress topology validation +- OpenHCS tree construction and presentation +- OpenHCS server/process actions + +Generic widget infrastructure (tree rebuild/state sync/poll scheduling) remains +owned by ``pyqt-reactive`` and is documented there. + +OpenHCS Browser Components +-------------------------- + +- ``ProgressTopologyState``: + validates worker/well ownership claims from progress events. +- ``ProgressTreeBuilder``: + builds ``ProgressNode`` trees with recursive aggregation policies. +- ``ServerRowPresenter``: + type-dispatched rendering for execution/viewer/generic servers. +- ``ServerTreePopulation``: + composes live scan results, launching viewers, and busy fallback rows. +- ``ServerKillService``: + kill-plan execution (graceful/force) with logging hooks. + +Canonical Tree Path +------------------- + +Execution subtree path is invariant: + +``plate -> worker -> well -> step`` + +Compilation path is plate -> compilation entries until execution events begin. + +Aggregation Semantics +--------------------- + +Tree percentages are recursive and policy-driven: + +- ``mean`` for parent aggregates (plate, worker) +- ``explicit`` for leaf/detail nodes (well, step, compilation) + +Policies are enforced per node type in ``ProgressTreeBuilder``. + +Refresh/State Preservation +-------------------------- + +OpenHCS coalesces progress updates on a timer and only rebuilds when dirty. +Expansion/selection preservation is delegated to the generic browser base. + +Primary Modules +--------------- + +- ``openhcs/pyqt_gui/widgets/shared/zmq_server_manager.py`` +- ``openhcs/pyqt_gui/widgets/shared/server_browser/progress_tree_builder.py`` +- ``openhcs/pyqt_gui/widgets/shared/server_browser/presentation_models.py`` +- ``openhcs/pyqt_gui/widgets/shared/server_browser/server_tree_population.py`` +- ``openhcs/pyqt_gui/widgets/shared/server_browser/server_kill_service.py`` + +See Also +-------- + +- :doc:`batch_workflow_service` +- :doc:`progress_runtime_projection_system` diff --git a/docs/source/concepts/component_system_concepts.rst b/docs/source/concepts/component_system_concepts.rst index c7666b692..397b795b3 100644 --- a/docs/source/concepts/component_system_concepts.rst +++ b/docs/source/concepts/component_system_concepts.rst @@ -112,6 +112,11 @@ Use GroupBy for dictionary function patterns and conditional routing: variable_components=[VariableComponents.SITE] ) +.. seealso:: + + :doc:`../architecture/pattern_grouping_and_special_outputs` + Comprehensive explanation of how ``group_by`` works for both dict and list patterns, including special output namespacing + **GroupBy properties**: .. code-block:: python diff --git a/docs/source/concepts/function_patterns.rst b/docs/source/concepts/function_patterns.rst index c37803488..40a1b2eab 100644 --- a/docs/source/concepts/function_patterns.rst +++ b/docs/source/concepts/function_patterns.rst @@ -3,6 +3,11 @@ Function Patterns OpenHCS provides four distinct function patterns that allow you to organize processing logic in different ways. Understanding these patterns is crucial for building effective analysis workflows that handle complex multi-channel, multi-condition experiments. +.. seealso:: + + :doc:`../architecture/pattern_grouping_and_special_outputs` + Deep dive into pattern grouping mechanics, special output path resolution, and the dual purpose of ``group_by`` + The Four Function Patterns -------------------------- diff --git a/docs/source/development/compact_window_patterns.rst b/docs/source/development/compact_window_patterns.rst new file mode 100644 index 000000000..7af70a47e --- /dev/null +++ b/docs/source/development/compact_window_patterns.rst @@ -0,0 +1,225 @@ +Compact Window Patterns +======================= + +Compact window patterns enable minimal UI layouts that work well on smaller screens and when tiling multiple windows side-by-side. + +Overview +-------- + +Traditional form dialogs often require large minimum widths to accommodate titles, help buttons, and action buttons in a single row. The compact window pattern separates these elements into multiple rows, allowing windows to be much narrower while maintaining usability. + +Two-Row Header Pattern +---------------------- + +The two-row header pattern separates the window title from action buttons: + +**Row 1**: Title and help button +**Row 2**: Action buttons (Save, Cancel, Reset, etc.) + +Traditional Single-Row Layout:: + + [Configure MyClass] [?] [View Code] [Reset] [Cancel] [Save] + + Minimum width: 600-800px + +Compact Two-Row Layout:: + + [Configure MyClass] [?] + [View Code] [Reset] [Cancel] [Save] + + Minimum width: 150-400px + +Implementation +-------------- + +Using _create_compact_header Helper +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``BaseManagedWindow`` class provides a helper for creating compact headers: + +.. code-block:: python + + from pyqt_reactive.widgets.shared.base_form_dialog import BaseManagedWindow + + class MyConfigWindow(BaseManagedWindow): + def setup_ui(self): + layout = QVBoxLayout(self) + + # Create compact two-row header + title_label, button_layout = self._create_compact_header( + layout, + title_text="Configure MyClass", + title_color=self.color_scheme.to_hex( + self.color_scheme.text_accent + ) + ) + + # Add action buttons to button row + save_btn = QPushButton("Save") + cancel_btn = QPushButton("Cancel") + button_layout.addWidget(save_btn) + button_layout.addWidget(cancel_btn) + +Manual Implementation +~~~~~~~~~~~~~~~~~~~~~ + +You can also implement the pattern manually: + +.. code-block:: python + + def setup_ui(self): + layout = QVBoxLayout(self) + + # Row 1: Title + title_widget = QWidget() + title_layout = QHBoxLayout(title_widget) + + self.header_label = QLabel("Window Title") + self.header_label.setFont(QFont("Arial", 14, QFont.Weight.Bold)) + title_layout.addWidget(self.header_label) + title_layout.addStretch() + + layout.addWidget(title_widget) + + # Row 2: Buttons + button_widget = QWidget() + button_layout = QHBoxLayout(button_widget) + button_layout.addStretch() + + save_btn = QPushButton("Save") + cancel_btn = QPushButton("Cancel") + button_layout.addWidget(save_btn) + button_layout.addWidget(cancel_btn) + + layout.addWidget(button_widget) + +Window Size Guidelines +---------------------- + +Minimum Sizes +~~~~~~~~~~~~~ + +For compact windows, use these minimum sizes: + +- **Minimum width**: 150px (ultra-minimal, allows narrow tiling) +- **Default width**: 400-500px (comfortable for most content) +- **Minimum height**: 150px (just enough for header + some content) + +Example from ConfigWindow: + +.. code-block:: python + + self.setMinimumSize(150, 150) # Ultra minimal + self.resize(400, 400) # Default size + +Benefits +-------- + +Screen Space Efficiency +~~~~~~~~~~~~~~~~~~~~~~~ + +- **Side-by-side tiling**: Two windows fit comfortably on a 1920px wide monitor +- **Laptop screens**: Works well on smaller screens (1366px, 1440px) +- **Multi-monitor setups**: Allows dense window arrangements + +Improved Usability +~~~~~~~~~~~~~~~~~~ + +- **Reduced eye movement**: Buttons are closer to content +- **Better focus**: Title is prominent at top +- **Scannable layout**: Clear visual hierarchy + +Integration with Responsive Widgets +----------------------------------- + +Combine compact headers with responsive layouts for maximum flexibility: + +.. code-block:: python + + from pyqt_reactive.widgets.shared.responsive_layout_widgets import ( + set_wrapping_enabled + ) + + # Enable responsive wrapping + set_wrapping_enabled(True) + + # Create compact window + window = ConfigWindow(config_class=MyConfig) + + # Parameter rows will wrap when narrow + # Window can be very narrow while remaining usable + +Examples +-------- + +ConfigWindow +~~~~~~~~~~~~ + +.. code-block:: python + + class ConfigWindow(BaseFormDialog): + def setup_ui(self): + self.setMinimumSize(150, 150) + self.resize(400, 400) + + layout = QVBoxLayout(self) + + # Compact two-row header + title_text = f"Configure {self.config_class.__name__}" + title_color = self.color_scheme.to_hex( + self.color_scheme.text_accent + ) + self._header_label, button_layout = self._create_compact_header( + layout, title_text, title_color + ) + + # Add help button to title row + help_btn = HelpButton(help_target=self.config_class) + title_row = self._header_label.parent().layout() + title_row.insertWidget(1, help_btn) + + # Add action buttons to button row + button_layout.addWidget(view_code_btn) + button_layout.addWidget(reset_btn) + button_layout.addWidget(cancel_btn) + button_layout.addWidget(save_btn) + +DualEditorWindow +~~~~~~~~~~~~~~~~ + +.. code-block:: python + + class DualEditorWindow(BaseFormDialog): + def setup_ui(self): + self.setMinimumSize(150, 150) + self.resize(400, 400) + + layout = QVBoxLayout(self) + + # Row 1: Title only + title_widget = QWidget() + title_layout = QHBoxLayout(title_widget) + + self.header_label = QLabel() + self.header_label.setFont( + QFont("Arial", 14, QFont.Weight.Bold) + ) + title_layout.addWidget(self.header_label) + title_layout.addStretch() + + layout.addWidget(title_widget) + + # Row 2: Tabs and buttons + control_row = QHBoxLayout() + control_row.addWidget(self.tab_bar, 0) + control_row.addStretch() + control_row.addWidget(cancel_btn) + control_row.addWidget(save_btn) + + layout.addLayout(control_row) + +See Also +-------- + +- :doc:`/architecture/parameter_form_service_architecture` - Form architecture +- :doc:`window_manager_usage` - Multi-window management diff --git a/docs/source/development/index.rst b/docs/source/development/index.rst index 0d7184a3f..bfd7fea23 100644 --- a/docs/source/development/index.rst +++ b/docs/source/development/index.rst @@ -36,6 +36,7 @@ Practical guides for specific development tasks. lazy_dataclass_utils pyclesperanto_simple_implementation window_manager_usage + compact_window_patterns Testing and CI ============== diff --git a/docs/source/development/respecting_codebase_architecture.rst b/docs/source/development/respecting_codebase_architecture.rst index 0f04b57df..f9af67539 100644 --- a/docs/source/development/respecting_codebase_architecture.rst +++ b/docs/source/development/respecting_codebase_architecture.rst @@ -68,14 +68,25 @@ Smart implementation respects codebase architecture by reusing information that .. code-block:: python - # RESPECTFUL CODE - reuse information from where it's already available - def initialize_step_plans_for_context(context, steps_definition, orchestrator): - # Config already retrieved here - effective_config = orchestrator.get_effective_config(for_serialization=False) - set_current_global_config(GlobalPipelineConfig, effective_config) - - # Add visualizer config while we have it - context.visualizer_config = effective_config.visualizer + # RESPECTFUL CODE - reuse information from where it's already available + def initialize_step_plans_for_context(context, steps_definition, orchestrator): + # Config already retrieved here + effective_config = orchestrator.get_effective_config(for_serialization=False) + set_current_global_config(GlobalPipelineConfig, effective_config) + + # Add visualizer config while we have it + context.visualizer_config = effective_config.visualizer + + # Modern compiler contract: reuse the resolved steps without rerunning path planning + resolved_steps, step_state_map = PipelineCompiler.initialize_step_plans_for_context( + context, + steps_definition, + orchestrator, + steps_already_resolved=False, + ) + + # ``step_state_map`` keeps the saved configuration snapshot for each step. + return resolved_steps, step_state_map **Key Principle**: When information is already available in a method's scope, use it there rather than calling the same method again elsewhere. This shows understanding of the data flow and prevents redundant operations. diff --git a/docs/source/development/window_manager_usage.rst b/docs/source/development/window_manager_usage.rst index 2be80c558..ef271fd03 100644 --- a/docs/source/development/window_manager_usage.rst +++ b/docs/source/development/window_manager_usage.rst @@ -1,5 +1,5 @@ Window Manager Usage Guide -========================== +========================= Overview -------- @@ -13,6 +13,30 @@ Key features: - Navigation API for focusing and scrolling to fields. - Fail-loud on stale references. +ServiceRegistry Integration +------------------------- + +Window handlers use ``ServiceRegistry`` for widget access instead of manual traversal: + +.. code-block:: python + + from pyqt_reactive.services import ServiceRegistry + from my_widgets import PlateManagerWidget + + def create_plate_config_window(scope_id: str, object_state=None): + # Get plate manager from ServiceRegistry + plate_manager = ServiceRegistry.get(PlateManagerWidget) + if not plate_manager: + return None + + window = ConfigWindow( + config_class=PipelineConfig, + current_config=orchestrator.pipeline_config, + scope_id=scope_id, + ) + window.show() + return window + Basic Usage ----------- @@ -297,3 +321,195 @@ Benefits 4. Extensible: navigation API ready for inheritance tracking. 5. Fail-loud: catches deleted windows early. 6. Fits OpenHCS patterns: similar to ``ObjectStateRegistry`` for states. + +Declarative Window Specifications +--------------------------------- + +For simple windows that don't require the full BaseFormDialog machinery, use the **WindowSpec** pattern with **ManagedWindow** classes. + +WindowSpec Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Define window specifications declaratively in your main window: + +.. code-block:: python + + from openhcs.pyqt_gui.services.window_config import WindowSpec + + class OpenHCSMainWindow(QMainWindow): + def __init__(self): + super().__init__() + self.window_specs = self._get_window_specs() + + def _get_window_specs(self) -> dict[str, WindowSpec]: + """Return declarative window specifications.""" + from openhcs.pyqt_gui.windows.managed_windows import ( + PlateManagerWindow, + PipelineEditorWindow, + ImageBrowserWindow, + ) + + return { + "plate_manager": WindowSpec( + window_id="plate_manager", + title="Plate Manager", + window_class=PlateManagerWindow, + initialize_on_startup=True, + ), + "pipeline_editor": WindowSpec( + window_id="pipeline_editor", + title="Pipeline Editor", + window_class=PipelineEditorWindow, + ), + "image_browser": WindowSpec( + window_id="image_browser", + title="Image Browser", + window_class=ImageBrowserWindow, + ), + } + +ManagedWindow Implementation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create a ManagedWindow class for each window type: + +.. code-block:: python + + # In openhcs/pyqt_gui/windows/managed_windows.py + + class PlateManagerWindow(QDialog): + """Window wrapper for PlateManagerWidget.""" + + def __init__(self, main_window, service_adapter): + super().__init__(main_window) + self.main_window = main_window + self.service_adapter = service_adapter + self.setWindowTitle("Plate Manager") + self.setModal(False) + self.resize(600, 400) + + # Create and add widget + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + layout = QVBoxLayout(self) + self.widget = PlateManagerWidget( + self.service_adapter, + self.service_adapter.get_current_color_scheme(), + ) + layout.addWidget(self.widget) + + # Setup signal connections + self._setup_connections() + + def _setup_connections(self): + """Connect signals to main window and other windows.""" + # Connect to main window + self.widget.global_config_changed.connect( + lambda: self.main_window.on_config_changed( + self.service_adapter.get_global_config() + ) + ) + + # Connect progress signals to status bar + if hasattr(self.main_window, "status_bar"): + self._setup_progress_signals() + + # Connect to other windows via WindowManager + self._connect_to_pipeline_editor() + +Showing Windows with WindowManager +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use WindowManager with a factory function: + +.. code-block:: python + + from pyqt_reactive.services.window_manager import WindowManager + + def show_window(self, window_id: str) -> None: + """Show window using WindowManager.""" + factory = self._create_window_factory(window_id) + window = WindowManager.show_or_focus(window_id, factory) + + # Optional: Initialize hidden for startup windows + spec = self.window_specs[window_id] + if spec.initialize_on_startup and window_id == "log_viewer": + window.hide() + + # Ensure flash overlay is ready + self._ensure_flash_overlay(window) + + def _create_window_factory(self, window_id: str) -> Callable[[], QDialog]: + """Create factory function for a window.""" + spec = self.window_specs[window_id] + + def factory() -> QDialog: + return spec.window_class(self, self.service_adapter) + + return factory + +Window-to-Window Communication +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +ManagedWindows can communicate via WindowManager: + +.. code-block:: python + + def _connect_to_pipeline_editor(self): + """Connect plate manager to pipeline editor.""" + from pyqt_reactive.services.window_manager import WindowManager + + # Get pipeline window if it exists + pipeline_window = WindowManager._scoped_windows.get("pipeline_editor") + if pipeline_window: + # Access the widget and connect signals + pipeline_widget = pipeline_window.widget + self.widget.plate_selected.connect(pipeline_widget.set_current_plate) + self.widget.orchestrator_config_changed.connect( + pipeline_widget.on_orchestrator_config_changed + ) + +When to Use Each Pattern +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Use WindowSpec + ManagedWindow for:** + +- Simple container windows (Plate Manager, Image Browser, etc.) +- Windows that don't need ObjectState/form management +- Windows that wrap existing widgets +- Quick prototyping + +**Use BaseFormDialog for:** + +- Configuration dialogs (ConfigWindow, DualEditorWindow) +- Forms with ParameterFormManager +- Windows that need ObjectState integration +- Complex multi-tab dialogs + +Comparison +~~~~~~~~~~~ + +.. list-table:: Window Management Patterns + :header-rows: 1 + :widths: 25 35 40 + + * - Feature + - WindowSpec + ManagedWindow + - BaseFormDialog + * - Complexity + - Low + - High + * - ObjectState + - Manual + - Automatic + * - Form Management + - No + - Built-in + * - Singleton Enforcement + - Via WindowManager + - Via WindowManager + * - Save/Restore + - Manual + - Automatic + * - Best For + - Simple containers + - Complex config dialogs diff --git a/docs/source/guide_for_biologists/configuration_reference.rst b/docs/source/guide_for_biologists/configuration_reference.rst index 345c31816..8d37d438d 100644 --- a/docs/source/guide_for_biologists/configuration_reference.rst +++ b/docs/source/guide_for_biologists/configuration_reference.rst @@ -25,16 +25,24 @@ GlobalPipelineConfig (main app / pipeline defaults) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - ``num_workers`` - - How many parallel workers to run for processing (higher = more CPU usage, but faster run times). + + How many parallel workers to run for processing (higher = more CPU usage, but faster run times). - ``materialization_results_path`` - - Directory name where non-image analysis results (CSV/JSON) are written by default. + + Directory name where non-image analysis results (CSV/JSON) are written by default. - ``use_threading`` - - If true, use threads instead of processes (useful for some environments, don't touch this unless you know what you're doing). + + If true, use threads instead of processes (useful for some environments, don't touch this unless you know what you're doing). + +- ``auto_add_output_plate_to_plate_manager`` (abbreviation: ``auto_add_output_plate``) + + If true, when a plate run completes successfully, the computed output plate root (from path planning) is automatically added to Plate Manager as a new orchestrator if it is not already present. This allows immediate visualization of processed results without manual plate addition. + + **Environment variable**: ``OPENHCS_AUTO_ADD_OUTPUT_PLATE`` + + **Default**: ``False`` WellFilterConfig ~~~~~~~~~~~~~~~~~ diff --git a/docs/source/guides/pipeline_compilation_workflow.rst b/docs/source/guides/pipeline_compilation_workflow.rst index 1b9c4f446..037f8419f 100644 --- a/docs/source/guides/pipeline_compilation_workflow.rst +++ b/docs/source/guides/pipeline_compilation_workflow.rst @@ -26,19 +26,26 @@ The ``PipelineCompiler`` executes five sequential phases for each well: context = self.create_context(well_id) # 5-Phase Compilation - PipelineCompiler.initialize_step_plans_for_context( - context, pipeline_definition, metadata_writer=is_responsible, plate_path=self.plate_path + resolved_steps, step_state_map = PipelineCompiler.initialize_step_plans_for_context( + context, + pipeline_definition, + metadata_writer=is_responsible, + plate_path=self.plate_path, + steps_already_resolved=False, ) PipelineCompiler.declare_zarr_stores_for_context( - context, pipeline_definition, self + context, resolved_steps, self ) PipelineCompiler.plan_materialization_flags_for_context( - context, pipeline_definition, self + context, resolved_steps, self ) PipelineCompiler.validate_memory_contracts_for_context( - context, pipeline_definition, self + context, + resolved_steps, + self, + step_state_map=step_state_map, ) - PipelineCompiler.assign_gpu_resources_for_context(context) + PipelineCompiler.assign_gpu_resources_for_context(context, resolved_steps, self) context.freeze() compiled_contexts[well_id] = context @@ -52,7 +59,9 @@ Phase 1: Step Plan Initialization **Key Operations**: - Creates step plans for each pipeline step +- Registers global/orchestrator/step ``ObjectState`` scopes and resolves their saved values before metadata injection - Calls ``PipelinePathPlanner.prepare_pipeline_paths()`` for path resolution +- Returns ``(resolved_steps, step_state_map)`` so later phases can access the exact configuration snapshot used during resolution (streaming defaults, visualizer settings, etc.) - Handles special I/O path linking between steps - Sets up chainbreaker status for steps that break the pipeline flow diff --git a/docs/source/user_guide/llm_pipeline_generation.rst b/docs/source/user_guide/llm_pipeline_generation.rst index 70f6d7ff2..acda747c3 100644 --- a/docs/source/user_guide/llm_pipeline_generation.rst +++ b/docs/source/user_guide/llm_pipeline_generation.rst @@ -188,7 +188,7 @@ The system prompt provides comprehensive OpenHCS context to the LLM: ## Guidelines - Use FunctionStep for each processing operation - Specify function name and parameters - - Use appropriate memory backends (@numpy_memory, @cupy_memory) + - Use appropriate memory decorators (@numpy, @cupy, @pyclesperanto) - Include materialization for final outputs Generate executable Python code that creates an OpenHCS pipeline. @@ -196,12 +196,73 @@ The system prompt provides comprehensive OpenHCS context to the LLM: return prompt + def get_system_prompt(self, code_type: str = "pipeline") -> str: + """Return the runtime-generated system prompt for a given context. + + Args: + code_type: Type of code being generated ("pipeline" or "function") + + Returns: + System prompt tailored for the specific code type + """ + if code_type == "function": + return self._system_prompts.get("function", self.system_prompt) + return self._system_prompts.get("pipeline", self.system_prompt) + **System prompt components**: 1. **Example pipeline**: Working OpenHCS pipeline code 2. **Function library**: Available processing functions and signatures 3. **Guidelines**: Best practices for pipeline construction 4. **API documentation**: Core classes and patterns +5. **Context-aware prompts**: Different prompts for pipeline vs function generation + +Array Backend Handling +---------------------- + +The LLM now understands OpenHCS memory decorators and handles array backends automatically: + +**Memory Decorators**: + +- ``@numpy`` - Function accepts and returns NumPy arrays +- ``@cupy`` - Function accepts and returns CuPy GPU arrays +- ``@pyclesperanto`` - Function accepts and returns pyclesperanto GPU arrays + +**Key Rules for Generated Functions**: + +1. **First parameter MUST be named 'image'** - 3D array in (C, Y, X) a.k.a. (Z, Y, X) format +2. **Accept the decorator's declared input type** - Don't manually convert between backends +3. **Return the declared output type** - OpenHCS handles cross-step conversions automatically +4. **No manual backend conversions** - Don't use ``cp.asnumpy()``, ``cle.pull()``, etc. +5. **Decorator adds keyword-only args** - ``slice_by_slice`` and ``dtype_conversion`` (defaults to preserving input dtype) + +**Example Function Generation**: + +.. code-block:: python + + # CuPy function - no manual conversion needed + @cupy + def count_cells_cupy(image, min_area=50): + import cupy as cp + from cucim.skimage import measure + + labeled = measure.label(image > 0) + regions = measure.regionprops(labeled, intensity_image=image) + + stats_list = [] + masks = [] + for props in regions: + if props.area >= min_area: + stats_list.append({ + 'area': int(props.area), + 'centroid': tuple(props.centroid) + }) + masks.append(labeled == props.label) + + return image, stats_list, masks # Return CuPy array directly + +**Important**: The function returns the CuPy array directly. OpenHCS automatically handles +conversions between steps with different memory decorators. Chat Panel Integration ====================== diff --git a/external/ObjectState b/external/ObjectState index 3b904efa1..7cc12dc7f 160000 --- a/external/ObjectState +++ b/external/ObjectState @@ -1 +1 @@ -Subproject commit 3b904efa1f465b1c6ba4617b48a9c9a756f23d41 +Subproject commit 7cc12dc7f1415e4cf9391ef7b3334dbf876b61b4 diff --git a/external/PolyStore b/external/PolyStore index 2478a9a51..7950340ec 160000 --- a/external/PolyStore +++ b/external/PolyStore @@ -1 +1 @@ -Subproject commit 2478a9a519eb7b6301ca6f21104e67b0162234c5 +Subproject commit 7950340ec5507dfa5b79e0cef3c6dc4d875b53b2 diff --git a/external/arraybridge b/external/arraybridge index e2294d9d0..ed0ffd1df 160000 --- a/external/arraybridge +++ b/external/arraybridge @@ -1 +1 @@ -Subproject commit e2294d9d042d8cd69409e6cd3826a801ea693485 +Subproject commit ed0ffd1df6cfaef8b38888eb1a656baccc38ece4 diff --git a/external/metaclass-registry b/external/metaclass-registry index 54d99dfa5..1e61e0535 160000 --- a/external/metaclass-registry +++ b/external/metaclass-registry @@ -1 +1 @@ -Subproject commit 54d99dfa5ea26d6abf87987c84f152f2a80d5b19 +Subproject commit 1e61e0535fbf0fb7be0059d04b8e6e947b158f7a diff --git a/external/pycodify b/external/pycodify index b2254860f..7dcec1b1f 160000 --- a/external/pycodify +++ b/external/pycodify @@ -1 +1 @@ -Subproject commit b2254860f25a51af9a10e527cdbf1c0fbf2d5d76 +Subproject commit 7dcec1b1f0b512d4f8a079fdbf8b2fd19f82952e diff --git a/external/pyqt-reactive b/external/pyqt-reactive index 6af769e4a..0ca2e8335 160000 --- a/external/pyqt-reactive +++ b/external/pyqt-reactive @@ -1 +1 @@ -Subproject commit 6af769e4a8e5bbdd9a61331e12d8ed6793da7bf3 +Subproject commit 0ca2e8335b86cb79c6b8c9a9b394e4d4eae38c77 diff --git a/external/python-introspect b/external/python-introspect index 5f53d2115..e08cbc45e 160000 --- a/external/python-introspect +++ b/external/python-introspect @@ -1 +1 @@ -Subproject commit 5f53d21154e5ed07cc0b25d315ae69a7c8502f7f +Subproject commit e08cbc45ea159481d605d1edca0e2454019aef83 diff --git a/external/zmqruntime b/external/zmqruntime index 66092356c..a189d35c8 160000 --- a/external/zmqruntime +++ b/external/zmqruntime @@ -1 +1 @@ -Subproject commit 66092356c2b29dda01fbc6003661e3ed584b8024 +Subproject commit a189d35c89eb657bf483183c68d94f017f032fce diff --git a/hooks/post-checkout b/hooks/post-checkout new file mode 100755 index 000000000..9a3d32257 --- /dev/null +++ b/hooks/post-checkout @@ -0,0 +1,45 @@ +#!/bin/bash +# Update submodules individually - update clean ones, skip dirty ones +# Then checkout the tracking branch to avoid detached HEAD + +echo "Checking submodules..." + +# Get list of submodules and process +git submodule status | while IFS= read -r line; do + [ -z "$line" ] && continue + + # Parse: SHA path (version) + sha=$(echo "$line" | awk '{print $1}') + path=$(echo "$line" | awk '{print $2}') + name=$(basename "$path") + + # Remove leading + or - from SHA + sha=${sha#+} + sha=${sha#-} + + # Check if submodule is clean (no staged, unstaged, or untracked changes) + if (cd "$path" && git diff --quiet HEAD && git diff --cached --quiet HEAD && [ -z "$(git ls-files --others --exclude-standard)" ] 2>/dev/null); then + # Clean - try to update + current=$(cd "$path" && git rev-parse --short HEAD) + if git submodule update --remote "$path" >/dev/null 2>&1; then + new=$(cd "$path" && git rev-parse --short HEAD) + if [ "$current" != "$new" ]; then + echo "Updated: $name ($current → $new)" + fi + fi + + # Checkout the tracking branch to avoid detached HEAD + branch=$(cd "$path" && git config -f "$toplevel/.gitmodules" submodule."$name".branch 2>/dev/null || echo main) + if (cd "$path" && git checkout "$branch" >/dev/null 2>&1); then + # Pull latest if we're now on the branch + if (cd "$path" && git pull --ff-only >/dev/null 2>&1); then + : # Successfully pulled + fi + fi + else + # Dirty - skip + echo "Skipped: $name (uncommitted changes)" + fi +done + +echo "Done. Run 'git submodule status' to see current state." diff --git a/hooks/post-merge b/hooks/post-merge new file mode 100755 index 000000000..1609fa1ef --- /dev/null +++ b/hooks/post-merge @@ -0,0 +1,45 @@ +#!/bin/bash +# Update submodules to latest on their tracked branches after merge/pull +# This complements post-checkout which runs when switching branches + +echo "Updating submodules after merge..." + +# Get list of submodules and process +git submodule status | while IFS= read -r line; do + [ -z "$line" ] && continue + + # Parse: SHA path (version) + sha=$(echo "$line" | awk '{print $1}') + path=$(echo "$line" | awk '{print $2}') + name=$(basename "$path") + + # Remove leading + or - from SHA + sha=${sha#+} + sha=${sha#-} + + # Check if submodule is clean (no staged, unstaged, or untracked changes) + if (cd "$path" && git diff --quiet HEAD && git diff --cached --quiet HEAD && [ -z "$(git ls-files --others --exclude-standard)" ] 2>/dev/null); then + # Clean - try to update to latest on remote branch + current=$(cd "$path" && git rev-parse --short HEAD) + if git submodule update --remote "$path" >/dev/null 2>&1; then + new=$(cd "$path" && git rev-parse --short HEAD) + if [ "$current" != "$new" ]; then + echo "Updated: $name ($current → $new)" + fi + fi + + # Checkout the tracking branch to avoid detached HEAD + branch=$(cd "$path" && git config -f "$toplevel/.gitmodules" submodule."$name".branch 2>/dev/null || echo main) + if (cd "$path" && git checkout "$branch" >/dev/null 2>&1); then + # Pull latest if we're now on the branch + if (cd "$path" && git pull --ff-only >/dev/null 2>&1); then + : # Successfully pulled + fi + fi + else + # Dirty - skip + echo "Skipped: $name (uncommitted changes)" + fi +done + +echo "Done. Run 'git submodule status' to see current state." diff --git a/omero_openhcs/INSTALL.md b/omero_openhcs/INSTALL.md index 64b127e50..c25940d54 100644 --- a/omero_openhcs/INSTALL.md +++ b/omero_openhcs/INSTALL.md @@ -81,12 +81,9 @@ omero web restart ```bash # From openhcs repository root -python -m openhcs.runtime.execution_server \ - --omero-host localhost \ - --omero-user root \ - --omero-password omero-root-password \ - --omero-data-dir /OMERO/Files \ - --server-port 7777 +python -m openhcs.runtime.zmq_execution_server_launcher \ + --port 7777 \ + --persistent ``` ### 4. Test the Integration diff --git a/omero_openhcs/README.md b/omero_openhcs/README.md index 969d41b77..35512c035 100644 --- a/omero_openhcs/README.md +++ b/omero_openhcs/README.md @@ -45,11 +45,9 @@ omero web restart ```bash # On the OMERO server machine -python -m openhcs.runtime.execution_server \ - --omero-host localhost \ - --omero-user root \ - --omero-password omero-root-password \ - --server-port 7777 +python -m openhcs.runtime.zmq_execution_server_launcher \ + --port 7777 \ + --persistent ``` ## Usage @@ -91,4 +89,3 @@ omero web start --debug ## License MIT License - see LICENSE file for details. - diff --git a/openhcs/__init__.py b/openhcs/__init__.py index 3f0359caa..31d7e7c68 100644 --- a/openhcs/__init__.py +++ b/openhcs/__init__.py @@ -13,7 +13,7 @@ import platform from pathlib import Path -__version__ = "0.5.0" +__version__ = "0.5.15" # Configure polystore defaults for OpenHCS integration os.environ.setdefault("POLYSTORE_METADATA_FILENAME", "openhcs_metadata.json") diff --git a/openhcs/constants/streaming.py b/openhcs/constants/streaming.py deleted file mode 100644 index 88517c084..000000000 --- a/openhcs/constants/streaming.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Enums for streaming backends and visualizers. - -This module provides type-safe enums for data types and shape types -used in streaming backends and visualizers. -""" - -from enum import Enum - - -class StreamingDataType(Enum): - """Types of data that can be streamed to viewers.""" - IMAGE = 'image' - SHAPES = 'shapes' # For Napari shapes layer - POINTS = 'points' # For Napari points layer (e.g., skeleton tracings) - ROIS = 'rois' # For Fiji - - -class NapariShapeType(Enum): - """Napari shape types for ROI visualization.""" - POLYGON = 'polygon' - ELLIPSE = 'ellipse' - POINT = 'point' - LINE = 'line' - PATH = 'path' - RECTANGLE = 'rectangle' - diff --git a/openhcs/core/components/validation.py b/openhcs/core/components/validation.py index a94516e2a..59acef601 100644 --- a/openhcs/core/components/validation.py +++ b/openhcs/core/components/validation.py @@ -11,11 +11,12 @@ from dataclasses import dataclass from openhcs.components.framework import ComponentConfiguration +from openhcs.constants.constants import GroupBy logger = logging.getLogger(__name__) -T = TypeVar('T', bound=Enum) -U = TypeVar('U', bound=Enum) +T = TypeVar("T", bound=Enum) +U = TypeVar("U", bound=Enum) def convert_enum_by_value(source_enum: T, target_enum_class: Type[U]) -> Optional[U]: @@ -48,6 +49,7 @@ def convert_enum_by_value(source_enum: T, target_enum_class: Type[U]) -> Optiona @dataclass class ValidationResult: """Result of a validation operation.""" + is_valid: bool error_message: Optional[str] = None warnings: Optional[List[str]] = None @@ -56,51 +58,53 @@ class ValidationResult: class GenericValidator(Generic[T]): """ Generic validator for component-agnostic validation. - + This class replaces the hardcoded component-specific validation logic with a configurable system that works with any component configuration. """ - + def __init__(self, config: ComponentConfiguration[T]): """ Initialize the validator with a component configuration. - + Args: config: ComponentConfiguration for validation rules """ self.config = config - logger.debug(f"GenericValidator initialized for components: {[c.value for c in config.all_components]}") - + logger.debug( + f"GenericValidator initialized for components: {[c.value for c in config.all_components]}" + ) + def validate_step( self, variable_components: List[T], - group_by: Optional[Union[T, 'GroupBy']], + group_by: Optional[Union[T, "GroupBy"]], func_pattern: Any, - step_name: str + step_name: str, ) -> ValidationResult: """ Validate a step configuration using generic rules. - + Args: variable_components: List of variable components group_by: Optional group_by component func_pattern: Function pattern (callable, dict, or list) step_name: Name of the step for error reporting - + Returns: ValidationResult indicating success or failure """ try: # 1. Validate component combination self.config.validate_combination(variable_components, group_by) - + # 2. Validate dict pattern requirements if isinstance(func_pattern, dict) and not group_by: return ValidationResult( is_valid=False, - error_message=f"Dict pattern requires group_by in step '{step_name}'" + error_message=f"Dict pattern requires group_by in step '{step_name}'", ) - + # 3. Validate components are in remaining components (not multiprocessing axis) remaining_components = self.config.get_remaining_components() remaining_values = {comp.value for comp in remaining_components} @@ -109,44 +113,41 @@ def validate_step( if component.value not in remaining_values: return ValidationResult( is_valid=False, - error_message=f"Variable component {component.value} not available (multiprocessing axis: {self.config.multiprocessing_axis.value})" + error_message=f"Variable component {component.value} not available (multiprocessing axis: {self.config.multiprocessing_axis.value})", ) # Check group_by is valid if it's not None # Note: group_by can be an enum with .value = None, so check the value explicitly - if group_by is not None and group_by.value is not None and group_by.value not in remaining_values: + if ( + group_by is not None + and group_by.value is not None + and group_by.value not in remaining_values + ): return ValidationResult( is_valid=False, - error_message=f"Group_by component {group_by.value} not available (multiprocessing axis: {self.config.multiprocessing_axis.value})" + error_message=f"Group_by component {group_by.value} not available (multiprocessing axis: {self.config.multiprocessing_axis.value})", ) - + return ValidationResult(is_valid=True) - + except ValueError as e: - return ValidationResult( - is_valid=False, - error_message=str(e) - ) - + return ValidationResult(is_valid=False, error_message=str(e)) + def validate_dict_pattern_keys( - self, - func_pattern: Dict[str, Any], - group_by: T, - step_name: str, - orchestrator + self, func_pattern: Dict[str, Any], group_by: T, step_name: str, orchestrator ) -> ValidationResult: """ Validate that dict function pattern keys match available component keys. - + This validation ensures compile-time guarantee that dict patterns will work at runtime by checking that all dict keys exist in the actual component data. - + Args: func_pattern: Dict function pattern to validate group_by: GroupBy component specifying component type step_name: Name of the step containing the function orchestrator: Orchestrator for component key access - + Returns: ValidationResult indicating success or failure """ @@ -154,28 +155,36 @@ def validate_dict_pattern_keys( # Use enum objects directly - orchestrator now accepts VariableComponents available_keys = orchestrator.get_component_keys(group_by) available_keys_set = set(str(key) for key in available_keys) - + # Check each dict key against available keys pattern_keys = list(func_pattern.keys()) pattern_keys_set = set(str(key) for key in pattern_keys) - + # Try direct string match first missing_keys = pattern_keys_set - available_keys_set - + if missing_keys: # Try numeric conversion for better error reporting try: - available_numeric = {str(int(float(k))) for k in available_keys if str(k).replace('.', '').isdigit()} - pattern_numeric = {str(int(float(k))) for k in pattern_keys if str(k).replace('.', '').isdigit()} + available_numeric = { + str(int(float(k))) + for k in available_keys + if str(k).replace(".", "").isdigit() + } + pattern_numeric = { + str(int(float(k))) + for k in pattern_keys + if str(k).replace(".", "").isdigit() + } missing_numeric = pattern_numeric - available_numeric - + if missing_numeric: return ValidationResult( is_valid=False, error_message=( f"Function pattern keys {sorted(missing_numeric)} not found in available " f"{group_by.value} components {sorted(available_numeric)} for step '{step_name}'" - ) + ), ) except (ValueError, TypeError): # Fall back to string comparison @@ -184,29 +193,27 @@ def validate_dict_pattern_keys( error_message=( f"Function pattern keys {sorted(missing_keys)} not found in available " f"{group_by.value} components {sorted(available_keys_set)} for step '{step_name}'" - ) + ), ) - + return ValidationResult(is_valid=True) - + except Exception as e: return ValidationResult( is_valid=False, - error_message=f"Failed to validate dict pattern keys for {group_by.value}: {e}" + error_message=f"Failed to validate dict pattern keys for {group_by.value}: {e}", ) - + def validate_component_combination_constraint( - self, - variable_components: List[T], - group_by: Optional[T] + self, variable_components: List[T], group_by: Optional[T] ) -> ValidationResult: """ Validate the core constraint: group_by ∉ variable_components. - + Args: variable_components: List of variable components group_by: Optional group_by component - + Returns: ValidationResult indicating success or failure """ @@ -214,7 +221,4 @@ def validate_component_combination_constraint( self.config.validate_combination(variable_components, group_by) return ValidationResult(is_valid=True) except ValueError as e: - return ValidationResult( - is_valid=False, - error_message=str(e) - ) + return ValidationResult(is_valid=False, error_message=str(e)) diff --git a/openhcs/core/config.py b/openhcs/core/config.py index 86cc63e0c..a46dc58d2 100644 --- a/openhcs/core/config.py +++ b/openhcs/core/config.py @@ -7,18 +7,42 @@ """ import logging -import os # For a potentially more dynamic default for num_workers +import os # For a potentially more dynamic default for num_workers from dataclasses import dataclass, field from pathlib import Path -from typing import Optional, Union, Any, List +from typing import Optional, Union, Any, List, Annotated from enum import Enum from abc import ABC, abstractmethod -from openhcs.constants import Microscope, SequentialComponents, VirtualComponents, VariableComponents, GroupBy, DtypeConversion -from openhcs.constants.constants import Backend, LiteralDtype, get_default_variable_components, get_default_group_by +from openhcs.constants import ( + Microscope, + SequentialComponents, + VirtualComponents, + VariableComponents, + GroupBy, + DtypeConversion, +) +from openhcs.constants.constants import ( + Backend, + LiteralDtype, + get_default_variable_components, + get_default_group_by, +) +from metaclass_registry import AutoRegisterMeta from openhcs.constants.input_source import InputSource +from python_introspect import Enableable +from python_introspect.enableable import EnableableMeta # Import decorator for automatic decorator creation -from objectstate import auto_create_decorator + + +# Combined metaclass for StreamingConfig to support both Enableable and AutoRegisterMeta +class StreamingConfigMeta(EnableableMeta, AutoRegisterMeta): + """Combined metaclass supporting Enableable semantics and AutoRegisterMeta registration.""" + + pass + + +from objectstate import auto_create_decorator, abbreviation # Import platform-aware transport mode default # This must be imported here to avoid circular imports @@ -29,13 +53,16 @@ class ZarrCompressor(Enum): """Available compression algorithms for zarr storage.""" + BLOSC = "blosc" ZLIB = "zlib" LZ4 = "lz4" ZSTD = "zstd" NONE = "none" - def create_compressor(self, compression_level: int, shuffle: bool = True) -> Optional[Any]: + def create_compressor( + self, compression_level: int, shuffle: bool = True + ) -> Optional[Any]: """Create the actual zarr compressor instance. Args: @@ -51,7 +78,9 @@ def create_compressor(self, compression_level: int, shuffle: bool = True) -> Opt case ZarrCompressor.NONE: return None case ZarrCompressor.BLOSC: - return zarr.Blosc(cname='lz4', clevel=compression_level, shuffle=shuffle) + return zarr.Blosc( + cname="lz4", clevel=compression_level, shuffle=shuffle + ) case ZarrCompressor.ZLIB: return zarr.Zlib(level=compression_level) case ZarrCompressor.LZ4: @@ -62,12 +91,14 @@ def create_compressor(self, compression_level: int, shuffle: bool = True) -> Opt class ZarrChunkStrategy(Enum): """Chunking strategies for zarr arrays.""" + WELL = "well" # Single chunk per well (optimal for batch I/O) FILE = "file" # One chunk per file (better for random access) class MaterializationBackend(Enum): """Available backends for materialization (persistent storage only).""" + AUTO = "auto" ZARR = "zarr" DISK = "disk" @@ -76,12 +107,14 @@ class MaterializationBackend(Enum): class WellFilterMode(Enum): """Well filtering modes for selective materialization.""" + INCLUDE = "include" # Materialize only specified wells EXCLUDE = "exclude" # Materialize all wells except specified ones class NormalizationMethod(Enum): """Normalization methods for experimental analysis.""" + FOLD_CHANGE = "fold_change" # value / control_mean Z_SCORE = "z_score" # (value - control_mean) / control_std PERCENT_CONTROL = "percent_control" # (value / control_mean) * 100 @@ -89,16 +122,19 @@ class NormalizationMethod(Enum): class MicroscopeFormat(Enum): """Supported microscope formats for experimental analysis.""" + EDDU_CX5 = "EDDU_CX5" # ThermoFisher CX5 format EDDU_METAXPRESS = "EDDU_metaxpress" # Molecular Devices MetaXpress format class TransportMode(Enum): """ZMQ transport modes for local vs remote communication.""" + IPC = "ipc" # Inter-process communication (local only, no firewall prompts) TCP = "tcp" # Network sockets (supports remote, triggers firewall) +@abbreviation("gpc") @auto_create_decorator @dataclass(frozen=True) class GlobalPipelineConfig: @@ -107,7 +143,9 @@ class GlobalPipelineConfig: This object is intended to be instantiated at application startup and treated as immutable. """ - materialization_results_path: Path = field(default=Path("results"), metadata={'ui_hidden': True}) + materialization_results_path: Annotated[Path, abbreviation("results_path")] = field( + default=Path("results"), metadata={"ui_hidden": True} + ) """ Path for materialized analysis results (CSV, JSON files from special outputs). @@ -122,16 +160,29 @@ class GlobalPipelineConfig: by the sub_dir field in each step's step_materialization_config. """ - num_workers: int = 1 + num_workers: Annotated[int, abbreviation("W")] = 1 """Number of worker processes/threads for parallelizable tasks.""" - - microscope: Microscope = field(default=Microscope.AUTO, metadata={'ui_hidden': True}) + + microscope: Annotated[Microscope, abbreviation("scope")] = field( + default=Microscope.AUTO, metadata={"ui_hidden": True} + ) """Default microscope type for auto-detection.""" - #use_threading: bool = field(default_factory=lambda: os.getenv('OPENHCS_USE_THREADING', 'false').lower() == 'true') - use_threading: bool = field(default_factory=lambda: os.getenv('OPENHCS_USE_THREADING', 'false').lower() == 'true', metadata={'ui_hidden': True}) + # use_threading: bool = field(default_factory=lambda: os.getenv('OPENHCS_USE_THREADING', 'false').lower() == 'true') + use_threading: Annotated[bool, abbreviation("threading")] = field( + default_factory=lambda: os.getenv("OPENHCS_USE_THREADING", "false").lower() + == "true", + metadata={"ui_hidden": True}, + ) """Use ThreadPoolExecutor instead of ProcessPoolExecutor for debugging. Reads from OPENHCS_USE_THREADING environment variable.""" + auto_add_output_plate_to_plate_manager: Annotated[ + bool, abbreviation("auto_add_output_plate") + ] = False + """If True, when a plate run completes successfully, the computed output plate root + (from path planning) is automatically added to Plate Manager as a new orchestrator + if it is not already present.""" + # Future extension point: # logging_config: Optional[Dict[str, Any]] = None # For configuring logging levels, handlers # plugin_settings: Dict[str, Any] = field(default_factory=dict) # For plugin-specific settings @@ -140,17 +191,18 @@ class GlobalPipelineConfig: # PipelineConfig will be created automatically by the injection system # (GlobalPipelineConfig → PipelineConfig by removing "Global" prefix) -# Register field abbreviations for GlobalPipelineConfig (can't use decorator on @auto_create_decorator) -from openhcs.config_framework.lazy_factory import FIELD_ABBREVIATIONS_REGISTRY -FIELD_ABBREVIATIONS_REGISTRY[GlobalPipelineConfig] = {'num_workers': 'W'} - # Import utilities for dynamic config creation from openhcs.utils.enum_factory import create_colormap_enum -from openhcs.utils.display_config_factory import create_napari_display_config, create_fiji_display_config +from openhcs.utils.display_config_factory import ( + create_napari_display_config, + create_fiji_display_config, +) # Import component order builder from factory module -from openhcs.core.streaming_config_factory import build_component_order as _build_component_order +from openhcs.core.streaming_config_factory import ( + build_component_order as _build_component_order, +) # Create colormap enum with minimal set to avoid importing napari (→ dask → GPU libs) # The lazy=True parameter uses a hardcoded minimal set instead of introspecting napari @@ -159,19 +211,25 @@ class GlobalPipelineConfig: class NapariDimensionMode(Enum): """How to handle different dimensions in napari visualization.""" + SLICE = "slice" # Show as 2D slice (take middle slice) STACK = "stack" # Show as 3D stack/volume class NapariVariableSizeHandling(Enum): """How to handle images with different sizes in the same layer.""" - SEPARATE_LAYERS = "separate_layers" # Create separate layers per well (preserves exact data) + + SEPARATE_LAYERS = ( + "separate_layers" # Create separate layers per well (preserves exact data) + ) PAD_TO_MAX = "pad_to_max" # Pad smaller images to match largest (enables stacking) # Visualization dtype normalization (alias to LiteralDtype - no duplication) VisualizationDtype = LiteralDtype -VisualizationDtype.__doc__ = """Dtype normalization for visualization streaming (Napari/Fiji stacking).""" +VisualizationDtype.__doc__ = ( + """Dtype normalization for visualization streaming (Napari/Fiji stacking).""" +) # Create NapariDisplayConfig using factory @@ -186,8 +244,8 @@ class NapariVariableSizeHandling(Enum): virtual_components=VirtualComponents, component_order=_build_component_order(), # Auto-generated from VirtualComponents virtual_component_defaults={ - 'source': NapariDimensionMode.SLICE # Separate layers per step by default - } + "source": NapariDimensionMode.SLICE # Separate layers per step by default + }, ) # Apply the global pipeline config decorator with ui_hidden=True @@ -199,8 +257,10 @@ class NapariVariableSizeHandling(Enum): # Fiji Display Configuration # ============================================================================ + class FijiLUT(Enum): """Fiji/ImageJ LUT options.""" + GRAYS = "Grays" FIRE = "Fire" ICE = "Ice" @@ -222,6 +282,7 @@ class FijiDimensionMode(Enum): - SLICE: Map to ImageJ Slice dimension (Z) - FRAME: Map to ImageJ Frame dimension (T) """ + WINDOW = "window" # Separate windows (like Napari SLICE mode) CHANNEL = "channel" # ImageJ Channel dimension (C) SLICE = "slice" # ImageJ Slice dimension (Z) @@ -238,8 +299,8 @@ class FijiDimensionMode(Enum): component_order=_build_component_order(), # Auto-generated from VirtualComponents virtual_component_defaults={ # source is WINDOW by default for window grouping (well is already WINDOW in component_defaults) - 'source': FijiDimensionMode.WINDOW - } + "source": FijiDimensionMode.WINDOW + }, ) # Apply the global pipeline config decorator with ui_hidden=True @@ -249,17 +310,24 @@ class FijiDimensionMode(Enum): FijiDisplayConfig._ui_hidden = True -@global_pipeline_config(field_abbreviations={'well_filter': 'wf', 'well_filter_mode': 'wfm'}) +@abbreviation("wfc") +@global_pipeline_config @dataclass(frozen=True) class WellFilterConfig: """Base configuration for well filtering functionality.""" - well_filter: Optional[Union[List[str], str, int]] = None + + well_filter: Annotated[Optional[Union[List[str], str, int]], abbreviation("")] = ( + None + ) """Well filter specification: list of wells, pattern string, or max count integer. None means all wells.""" - well_filter_mode: WellFilterMode = WellFilterMode.INCLUDE + well_filter_mode: Annotated[WellFilterMode, abbreviation("filter_mode")] = ( + WellFilterMode.INCLUDE + ) """Whether well_filter is an include list or exclude list.""" +@abbreviation("zarr") @global_pipeline_config @dataclass(frozen=True) class ZarrConfig: @@ -268,54 +336,77 @@ class ZarrConfig: OME-ZARR metadata and plate metadata are always enabled for HCS compliance. Shuffle filter is always enabled for Blosc compressor (ignored for others). """ - compressor: ZarrCompressor = ZarrCompressor.ZLIB + + compressor: Annotated[ZarrCompressor, abbreviation("compressor")] = ( + ZarrCompressor.ZLIB + ) """Compression algorithm to use.""" - compression_level: int = 3 + compression_level: Annotated[int, abbreviation("level")] = 3 """Compression level (1-9 for LZ4, higher = more compression).""" - chunk_strategy: ZarrChunkStrategy = ZarrChunkStrategy.WELL + chunk_strategy: Annotated[ZarrChunkStrategy, abbreviation("chunks")] = ( + ZarrChunkStrategy.WELL + ) """Chunking strategy: WELL (single chunk per well) or FILE (one chunk per file).""" -@global_pipeline_config(field_abbreviations={'materialization_backend': 'mat', 'intermediate_backend': 'int', 'read_backend': 'read'}) +@abbreviation("vfs") +@global_pipeline_config @dataclass(frozen=True) class VFSConfig: """Configuration for Virtual File System (VFS) related operations.""" - read_backend: Backend = Backend.AUTO + + read_backend: Annotated[Backend, abbreviation("read")] = Backend.AUTO """Backend for reading input data. AUTO uses metadata-based detection for OpenHCS plates.""" - intermediate_backend: Backend = Backend.MEMORY + intermediate_backend: Annotated[Backend, abbreviation("intermediate")] = ( + Backend.MEMORY + ) """Backend for storing intermediate step results that are not explicitly materialized.""" - materialization_backend: MaterializationBackend = MaterializationBackend.DISK + materialization_backend: Annotated[ + MaterializationBackend, abbreviation("materialize") + ] = MaterializationBackend.DISK """Backend for explicitly materialized outputs (e.g., final results, user-requested saves).""" +@abbreviation("dtype") @global_pipeline_config @dataclass(frozen=True) class DtypeConfig: """Configuration for dtype conversion behavior in memory type decorators.""" - default_dtype_conversion: DtypeConversion = DtypeConversion.NATIVE_OUTPUT + default_dtype_conversion: Annotated[DtypeConversion, abbreviation("conv")] = ( + DtypeConversion.NATIVE_OUTPUT + ) """Default dtype conversion mode for all decorated functions. NATIVE_OUTPUT (no scaling) or PRESERVE_INPUT (scale to input dtype).""" +@abbreviation("proc") @global_pipeline_config @dataclass(frozen=True) class ProcessingConfig: """Configuration for step processing behavior including variable components, grouping, and input source.""" - variable_components: List[VariableComponents] = field(default_factory=get_default_variable_components) + variable_components: Annotated[List[VariableComponents], abbreviation("vars")] = ( + field(default_factory=get_default_variable_components) + ) """List of variable components for pattern expansion.""" - group_by: Optional[GroupBy] = field(default_factory=get_default_group_by) + group_by: Annotated[Optional[GroupBy], abbreviation("group")] = field( + default_factory=get_default_group_by + ) """Component to group patterns by for conditional function routing.""" - input_source: InputSource = InputSource.PREVIOUS_STEP + input_source: Annotated[InputSource, abbreviation("source")] = ( + InputSource.PREVIOUS_STEP + ) """Input source strategy: PREVIOUS_STEP (normal chaining) or PIPELINE_START (access original input).""" + +@abbreviation("seq") @global_pipeline_config @dataclass(frozen=True) class SequentialProcessingConfig: @@ -326,92 +417,114 @@ class SequentialProcessingConfig: This is a pipeline-level setting, not per-step. """ - sequential_components: List[SequentialComponents] = field(default_factory=list) + sequential_components: Annotated[ + List[SequentialComponents], abbreviation("seq_comp") + ] = field(default_factory=list) """Components to process sequentially (e.g., [SequentialComponents.TIMEPOINT, SequentialComponents.CHANNEL]). When set, the orchestrator will process one combination of these components through all pipeline steps before moving to the next combination, clearing memory between combinations. """ + +@abbreviation("analysis") @global_pipeline_config @dataclass(frozen=True) -class AnalysisConsolidationConfig: - """Configuration for automatic analysis results consolidation.""" - enabled: bool = True - """Whether to automatically run analysis consolidation after pipeline completion.""" +class AnalysisConsolidationConfig(Enableable): + """Configuration for automatic analysis results consolidation. + + enabled controls whether consolidation runs after pipeline completion. + """ - metaxpress_style: bool = True + enabled: Annotated[bool, abbreviation("")] = True + + metaxpress_style: Annotated[bool, abbreviation("mx_style")] = True """Whether to generate MetaXpress-compatible output format with headers.""" - well_pattern: str = r"([A-Z]\d{2})" + well_pattern: Annotated[str, abbreviation("well_pat")] = r"([A-Z]\d{2})" """Regex pattern for extracting well IDs from filenames.""" - file_extensions: tuple[str, ...] = (".csv",) + file_extensions: Annotated[tuple[str, ...], abbreviation("exts")] = (".csv",) """File extensions to include in consolidation.""" - exclude_patterns: tuple[str, ...] = (r".*consolidated.*", r".*metaxpress.*", r".*summary.*") + exclude_patterns: Annotated[tuple[str, ...], abbreviation("exclude")] = ( + r".*consolidated.*", + r".*metaxpress.*", + r".*summary.*", + ) """Filename patterns to exclude from consolidation.""" - output_filename: str = "metaxpress_style_summary.csv" + output_filename: Annotated[str, abbreviation("out_file")] = ( + "metaxpress_style_summary.csv" + ) """Name of the consolidated output file.""" - global_summary_filename: str = "global_metaxpress_summary.csv" + global_summary_filename: Annotated[str, abbreviation("global_sum")] = ( + "global_metaxpress_summary.csv" + ) """Name of the global consolidated summary file combining all plates.""" +@abbreviation("plate") @global_pipeline_config @dataclass(frozen=True) class PlateMetadataConfig: """Configuration for plate metadata in MetaXpress-style output.""" - barcode: Optional[str] = None + + barcode: Annotated[Optional[str], abbreviation("barcode")] = None """Plate barcode. If None, will be auto-generated from plate name.""" - plate_name: Optional[str] = None + plate_name: Annotated[Optional[str], abbreviation("name")] = None """Plate name. If None, will be derived from plate path.""" - plate_id: Optional[str] = None + plate_id: Annotated[Optional[str], abbreviation("id")] = None """Plate ID. If None, will be auto-generated.""" - description: Optional[str] = None + description: Annotated[Optional[str], abbreviation("description")] = None """Experiment description. If None, will be auto-generated.""" - acquisition_user: str = "OpenHCS" + acquisition_user: Annotated[str, abbreviation("user")] = "OpenHCS" """User who acquired the data.""" - z_step: str = "1" + z_step: Annotated[str, abbreviation("z_step")] = "1" """Z-step information for MetaXpress compatibility.""" +@abbreviation("exp") @global_pipeline_config @dataclass(frozen=True) class ExperimentalAnalysisConfig: """Configuration for experimental analysis system.""" - config_file_name: str = "config.xlsx" + + config_file_name: Annotated[str, abbreviation("config")] = "config.xlsx" """Name of the experimental configuration Excel file.""" - design_sheet_name: str = "drug_curve_map" + design_sheet_name: Annotated[str, abbreviation("design")] = "drug_curve_map" """Name of the sheet containing experimental design.""" - plate_groups_sheet_name: str = "plate_groups" + plate_groups_sheet_name: Annotated[str, abbreviation("groups")] = "plate_groups" """Name of the sheet containing plate group mappings.""" - normalization_method: NormalizationMethod = NormalizationMethod.FOLD_CHANGE + normalization_method: Annotated[NormalizationMethod, abbreviation("norm")] = ( + NormalizationMethod.FOLD_CHANGE + ) """Normalization method for control-based normalization.""" - export_raw_results: bool = True + export_raw_results: Annotated[bool, abbreviation("raw")] = True """Whether to export raw (non-normalized) results.""" - export_heatmaps: bool = True + export_heatmaps: Annotated[bool, abbreviation("heatmaps")] = True """Whether to generate heatmap visualizations.""" - auto_detect_format: bool = True + auto_detect_format: Annotated[bool, abbreviation("auto_format")] = True """Whether to automatically detect microscope format.""" - default_format: Optional[MicroscopeFormat] = None + default_format: Annotated[Optional[MicroscopeFormat], abbreviation("format")] = None """Default format to use if auto-detection fails.""" -@global_pipeline_config(field_abbreviations={'output_dir_suffix': 'out', 'global_output_folder': 'gof', 'sub_dir': 'sub'}) +@abbreviation("pp") +@global_pipeline_config @dataclass(frozen=True) class PathPlanningConfig(WellFilterConfig): """ @@ -423,10 +536,13 @@ class PathPlanningConfig(WellFilterConfig): Inherits well filtering functionality from WellFilterConfig. """ - output_dir_suffix: str = "_openhcs" + + output_dir_suffix: Annotated[str, abbreviation("suffix")] = "_openhcs" """Default suffix for general step output directories.""" - global_output_folder: Optional[Path] = None + global_output_folder: Annotated[Optional[Path], abbreviation("global_folder")] = ( + None + ) """ Optional global output folder where all plate workspaces and outputs will be created. If specified, plate workspaces will be created as {global_output_folder}/{plate_name}_workspace/ @@ -435,23 +551,28 @@ class PathPlanningConfig(WellFilterConfig): Example: "/data/results" or "/mnt/hcs_output" """ - sub_dir: str = "images" + sub_dir: Annotated[str, abbreviation("subdir")] = "images" """ Subdirectory within plate folder for storing processed data. Examples: "images", "processed", "data/images" """ + +@abbreviation("step_wf") @global_pipeline_config @dataclass(frozen=True) class StepWellFilterConfig(WellFilterConfig): """Well filter configuration specialized for step-level configs with different defaults.""" + # Override defaults for step-level configurations - #well_filter: Optional[Union[List[str], str, int]] = 1 + # well_filter: Optional[Union[List[str], str, int]] = 1 pass -@global_pipeline_config(preview_label='MAT') + +@abbreviation("mat") +@global_pipeline_config(preview_label="MAT", always_viewable_fields=["sub_dir"]) @dataclass(frozen=True) -class StepMaterializationConfig(StepWellFilterConfig, PathPlanningConfig): +class StepMaterializationConfig(Enableable, StepWellFilterConfig, PathPlanningConfig): """ Configuration for per-step materialization - configurable in UI. @@ -460,41 +581,66 @@ class StepMaterializationConfig(StepWellFilterConfig, PathPlanningConfig): materialization instances will inherit these defaults unless explicitly overridden. Uses multiple inheritance from PathPlanningConfig and StepWellFilterConfig. + + The 'sub_dir' field is conditionally shown in list item previews via always_viewable_fields. + Since this config is Enableable, the sub_dir will only appear when enabled=True. + This means disabled materialization configs won't clutter the preview with sub_dir. """ - #Override sub_dir for materialization-specific default - sub_dir: str = "checkpoints" + # Override sub_dir for materialization-specific default + sub_dir: Annotated[str, abbreviation("subdir")] = "checkpoints" """Subdirectory for materialized outputs (different from global 'images').""" - enabled: bool = False + enabled: Annotated[bool, abbreviation("enabled")] = False """Whether this materialization config is enabled. When False, config exists but materialization is disabled.""" # Define platform-aware default transport mode at module level # TCP on Windows (no Unix domain socket support), IPC on Unix/Mac -_DEFAULT_TRANSPORT_MODE = TransportMode.TCP if platform.system() == 'Windows' else TransportMode.IPC +_DEFAULT_TRANSPORT_MODE = ( + TransportMode.TCP if platform.system() == "Windows" else TransportMode.IPC +) -@global_pipeline_config +@abbreviation("stream") +@global_pipeline_config(always_viewable_fields=["well_filter"]) @dataclass(frozen=True) -class StreamingDefaults(StepWellFilterConfig): - """Default configuration for streaming to visualizers.""" - persistent: bool = True +class StreamingDefaults(Enableable, StepWellFilterConfig): + """Default configuration for streaming to visualizers. + + The 'persistent' field is conditionally shown in list item previews via + always_viewable_fields. Since this config is Enableable, the persistent field + will only appear when enabled=True. This means disabled streaming configs won't + clutter the preview with persistence info, but enabled ones will show whether + the viewer persists after pipeline completion. + """ + + persistent: Annotated[bool, abbreviation("persist")] = True """Whether viewer stays open after pipeline completion.""" - host: str = 'localhost' + host: Annotated[str, abbreviation("host")] = "localhost" """Host for streaming communication. Use 'localhost' for local, or remote IP for network streaming.""" - - transport_mode: TransportMode = _DEFAULT_TRANSPORT_MODE + transport_mode: Annotated[TransportMode, abbreviation("transport")] = ( + _DEFAULT_TRANSPORT_MODE + ) """ZMQ transport mode: Platform-aware default (TCP on Windows, IPC on Unix/Mac).""" - enabled: bool = False + enabled: Annotated[bool, abbreviation("enabled")] = False """Whether this streaming config is enabled. When False, config exists but streaming is disabled.""" + batch_size: Annotated[Optional[int], abbreviation("batch")] = None + """Number of images to batch before displaying. + + None = wait for all images in the current operation, then display once (fastest, default). + N = show incrementally every N images (provides visual feedback but slower). + """ + + +@abbreviation("stream_cfg") @global_pipeline_config(ui_hidden=True) @dataclass(frozen=True) -class StreamingConfig(StreamingDefaults, ABC): +class StreamingConfig(StreamingDefaults, ABC, metaclass=StreamingConfigMeta): """Abstract base configuration for streaming to visualizers. Uses multiple inheritance from StepWellFilterConfig and StreamingDefaults. @@ -502,6 +648,18 @@ class StreamingConfig(StreamingDefaults, ABC): by @global_pipeline_config(inherit_as_none=True), enabling polymorphic access without type-specific attribute names. """ + + # AutoRegisterMeta configuration - subclasses auto-register by snake_case class name + __registry_key__ = "_streaming_config_key" + __key_extractor__ = ( + lambda class_name, cls: __import__("re") + .sub(r"(? int: @@ -540,27 +698,29 @@ def create_visualizer(self, filemanager, visualizer_config): # Auto-generate streaming configs using factory (reduces ~110 lines to ~20 lines) from openhcs.core.streaming_config_factory import create_streaming_config -NapariStreamingConfig = create_streaming_config( - viewer_name='napari', - port=5555, - backend=Backend.NAPARI_STREAM, - display_config_class=NapariDisplayConfig, - visualizer_module='openhcs.runtime.napari_stream_visualizer', - visualizer_class_name='NapariStreamVisualizer', - preview_label='NAP' +NapariStreamingConfig = abbreviation("nap")( + create_streaming_config( + viewer_name="napari", + port=5555, + backend=Backend.NAPARI_STREAM, + display_config_class=NapariDisplayConfig, + visualizer_module="openhcs.runtime.napari_stream_visualizer", + visualizer_class_name="NapariStreamVisualizer", + preview_label="NAP", + ) ) -FijiStreamingConfig = create_streaming_config( - viewer_name='fiji', - port=5565, - backend=Backend.FIJI_STREAM, - display_config_class=FijiDisplayConfig, - visualizer_module='openhcs.runtime.fiji_stream_visualizer', - visualizer_class_name='FijiStreamVisualizer', - extra_fields={ - 'fiji_executable_path': (Optional[Path], None) - }, - preview_label='FIJI' +FijiStreamingConfig = abbreviation("fiji")( + create_streaming_config( + viewer_name="fiji", + port=5565, + backend=Backend.FIJI_STREAM, + display_config_class=FijiDisplayConfig, + visualizer_module="openhcs.runtime.fiji_stream_visualizer", + visualizer_class_name="FijiStreamVisualizer", + extra_fields={"fiji_executable_path": (Optional[Path], None)}, + preview_label="FIJI", + ) ) # Inject all accumulated fields at the end of module loading. @@ -568,6 +728,7 @@ def create_visualizer(self, filemanager, visualizer_config): # (objectstate.lazy_factory). Importing via openhcs.config_framework.lazy_factory # would create a second module with its own empty registry. from objectstate.lazy_factory import _inject_all_pending_fields + _inject_all_pending_fields() diff --git a/openhcs/core/context/processing_context.py b/openhcs/core/context/processing_context.py index f34b3e110..9d9ad6db8 100644 --- a/openhcs/core/context/processing_context.py +++ b/openhcs/core/context/processing_context.py @@ -61,6 +61,12 @@ def __init__( self.global_config = global_config # Store the global config self.filemanager = None # Expected to be set by Orchestrator via kwargs or direct assignment + # Execution tracking fields (set at execution time) + self.execution_id = None # Set by worker before execution + self.plate_id = None # Set by worker before execution (same as plate_path) + self.worker_slot = None # Logical worker slot for deterministic ownership + self.owned_wells = None # Full well set owned by this worker slot + # Pipeline-wide sequential processing fields self.pipeline_sequential_mode = False self.pipeline_sequential_combinations = None # Precomputed at compile time from metadata @@ -77,7 +83,7 @@ def __setattr__(self, name: str, value: Any) -> None: All fields are immutable once frozen - no exceptions. """ - if getattr(self, '_is_frozen', False) and name != '_is_frozen': + if self._is_frozen and name != '_is_frozen': raise AttributeError(f"Cannot modify attribute '{name}' of a frozen ProcessingContext.") super().__setattr__(name, value) diff --git a/openhcs/core/metadata_cache.py b/openhcs/core/metadata_cache.py index 60770b1d0..a53fd8276 100644 --- a/openhcs/core/metadata_cache.py +++ b/openhcs/core/metadata_cache.py @@ -13,13 +13,15 @@ class MetadataCache: """Stores component key→name mappings with basic invalidation and thread safety.""" - + def __init__(self): - self._cache: Dict['AllComponents', Dict[str, Optional[str]]] = {} + self._cache: Dict["AllComponents", Dict[str, Optional[str]]] = {} self._metadata_file_mtimes: Dict[Path, float] = {} self._lock = threading.Lock() - - def cache_metadata(self, microscope_handler, plate_path: Path, component_keys_cache: Dict) -> None: + + def cache_metadata( + self, microscope_handler, plate_path: Path, component_keys_cache: Dict + ) -> None: """Cache all metadata from metadata handler.""" with self._lock: # Parse all metadata once @@ -34,7 +36,9 @@ def cache_metadata(self, microscope_handler, plate_path: Path, component_keys_ca # Update with actual metadata where available for component_name, mapping in metadata.items(): component = AllComponents(component_name) - logger.info(f"🔍 METADATA_CACHE: Caching {component_name} -> {component}: {mapping}") + logger.info( + f"🔍 METADATA_CACHE: Caching {component_name} -> {component}: {mapping}" + ) if component in self._cache: combined_cache = self._cache[component].copy() for metadata_key in mapping.keys(): @@ -48,10 +52,14 @@ def cache_metadata(self, microscope_handler, plate_path: Path, component_keys_ca logger.info(f"🔍 METADATA_CACHE: Final cache state: {self._cache}") # Store metadata file mtime for invalidation - metadata_file = microscope_handler.metadata_handler.find_metadata_file(plate_path) + metadata_file = microscope_handler.metadata_handler.find_metadata_file( + plate_path + ) if metadata_file and metadata_file.exists(): - self._metadata_file_mtimes[metadata_file] = metadata_file.stat().st_mtime - + self._metadata_file_mtimes[metadata_file] = ( + metadata_file.stat().st_mtime + ) + def get_component_metadata(self, component, key: str) -> Optional[str]: """Get metadata display name for a component key. Accepts GroupBy or VariableComponents.""" with self._lock: @@ -61,36 +69,43 @@ def get_component_metadata(self, component, key: str) -> Optional[str]: return None # Convert GroupBy to AllComponents using OpenHCS generic utility - original_component = component component = convert_enum_by_value(component, AllComponents) or component - logger.info(f"🔍 METADATA_CACHE get_component_metadata: {original_component} -> {component}, key={key!r}") - logger.info(f"🔍 METADATA_CACHE: cache keys = {list(self._cache.keys())}") + component_cache = self._cache.get(component) + if component_cache is None: + logger.debug("🔍 METADATA_CACHE: component %s not cached", component) + return None - component_cache = self._cache.get(component, {}) - logger.info(f"🔍 METADATA_CACHE: component_cache for {component} = {component_cache}") + if key not in component_cache: + logger.debug( + "🔍 METADATA_CACHE: key %r not cached for %s", key, component + ) + return None + + return component_cache.get(key) - result = component_cache.get(key) - logger.info(f"🔍 METADATA_CACHE: result for key {key!r} = {result!r}") - return result - - def get_cached_metadata(self, component: 'AllComponents') -> Optional[Dict[str, Optional[str]]]: + def get_cached_metadata( + self, component: "AllComponents" + ) -> Optional[Dict[str, Optional[str]]]: """Get all cached metadata for a component.""" with self._lock: if not self._is_cache_valid(): self._cache.clear() return None return self._cache.get(component) - + def clear_cache(self) -> None: """Clear cached metadata.""" with self._lock: self._cache.clear() self._metadata_file_mtimes.clear() - + def _is_cache_valid(self) -> bool: """Check if cache is valid by comparing file mtimes.""" for metadata_file, cached_mtime in self._metadata_file_mtimes.items(): - if not metadata_file.exists() or metadata_file.stat().st_mtime != cached_mtime: + if ( + not metadata_file.exists() + or metadata_file.stat().st_mtime != cached_mtime + ): return False return True diff --git a/openhcs/core/orchestrator/orchestrator.py b/openhcs/core/orchestrator/orchestrator.py index 9a97198ef..5bb328a18 100644 --- a/openhcs/core/orchestrator/orchestrator.py +++ b/openhcs/core/orchestrator/orchestrator.py @@ -12,47 +12,56 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Union, Set -from openhcs.constants.constants import Backend, DEFAULT_IMAGE_EXTENSIONS, GroupBy, OrchestratorState, get_openhcs_config, AllComponents, VariableComponents +from openhcs.constants.constants import ( + Backend, + DEFAULT_IMAGE_EXTENSIONS, + GroupBy, + OrchestratorState, + get_openhcs_config, + AllComponents, + VariableComponents, +) from openhcs.constants import Microscope from openhcs.core.config import GlobalPipelineConfig from openhcs.config_framework.global_config import get_current_global_config - from openhcs.core.metadata_cache import get_metadata_cache, MetadataCache from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.pipeline.compiler import PipelineCompiler from openhcs.core.steps.abstract import AbstractStep from openhcs.core.components.validation import convert_enum_by_value from openhcs.core.orchestrator.execution_result import ExecutionResult, ExecutionStatus +from openhcs.core.progress import ( + emit, + set_progress_queue, + ProgressPhase, + ProgressStatus, + create_event, +) from polystore.filemanager import FileManager + # Zarr backend is CPU-only; always import it (even in subprocess/no-GPU mode) import os from polystore.zarr import ZarrStorageBackend + # PipelineConfig now imported directly above -from openhcs.config_framework.lazy_factory import resolve_lazy_configurations_for_serialization +from openhcs.config_framework.lazy_factory import ( + resolve_lazy_configurations_for_serialization, +) from openhcs.microscopes import create_microscope_handler from openhcs.microscopes.microscope_base import MicroscopeHandler +from openhcs.processing.backends.analysis.consolidate_analysis_results import ( + consolidate_results_directories, +) -# Lazy import of consolidate_analysis_results to avoid blocking GUI startup -# This function imports GPU libraries, so we defer it until first use -def _get_consolidate_analysis_results(): - """Lazy import of consolidate_analysis_results function.""" - if os.getenv('OPENHCS_SUBPROCESS_NO_GPU') == '1': - # Subprocess runner mode - create placeholder - def consolidate_analysis_results(*args, **kwargs): - """Placeholder for subprocess runner mode.""" - raise RuntimeError("Analysis consolidation not available in subprocess runner mode") - return consolidate_analysis_results - else: - from openhcs.processing.backends.analysis.consolidate_analysis_results import consolidate_analysis_results - return consolidate_analysis_results # Import generic component system - required for orchestrator functionality # Optional napari import for visualization try: from openhcs.runtime.napari_stream_visualizer import NapariStreamVisualizer + NapariVisualizerType = NapariStreamVisualizer except ImportError: # Create a placeholder type for type hints when napari is not available @@ -97,7 +106,9 @@ def _merge_nested_dataclass(pipeline_value, global_value): if raw_pipeline_field is not None: # Pipeline has an explicitly set value - check if it's a nested dataclass that needs merging if is_dataclass(raw_pipeline_field) and is_dataclass(global_field_value): - merged_values[field.name] = _merge_nested_dataclass(raw_pipeline_field, global_field_value) + merged_values[field.name] = _merge_nested_dataclass( + raw_pipeline_field, global_field_value + ) else: merged_values[field.name] = raw_pipeline_field else: @@ -108,14 +119,18 @@ def _merge_nested_dataclass(pipeline_value, global_value): return type(pipeline_value)(**merged_values) -def _create_merged_config(pipeline_config: 'PipelineConfig', global_config: GlobalPipelineConfig) -> GlobalPipelineConfig: +def _create_merged_config( + pipeline_config: "PipelineConfig", global_config: GlobalPipelineConfig +) -> GlobalPipelineConfig: """ Pure function for creating merged config that preserves None values for sibling inheritance. Follows OpenHCS stateless architecture principles - no side effects, explicit dependencies. Extracted from apply_pipeline_config to eliminate code duplication. """ - logger.debug(f"Starting merge with pipeline_config={type(pipeline_config)} and global_config={type(global_config)}") + logger.debug( + f"Starting merge with pipeline_config={type(pipeline_config)} and global_config={type(global_config)}" + ) merged_config_values = {} for field in fields(GlobalPipelineConfig): @@ -128,15 +143,20 @@ def _create_merged_config(pipeline_config: 'PipelineConfig', global_config: Glob # CRITICAL FIX: For lazy configs, merge with global config BEFORE converting to base # This ensures None values in lazy configs resolve to global values # Then convert to base config to store in thread-local context - if hasattr(pipeline_value, 'to_base_config'): + if hasattr(pipeline_value, "to_base_config"): # This is a lazy config - merge with global config first global_value = getattr(global_config, field.name) from dataclasses import is_dataclass + if is_dataclass(global_value): # Merge lazy config with global config to resolve None values merged_lazy = _merge_nested_dataclass(pipeline_value, global_value) # Now convert merged result to base config - converted_value = merged_lazy.to_base_config() if hasattr(merged_lazy, 'to_base_config') else merged_lazy + converted_value = ( + merged_lazy.to_base_config() + if hasattr(merged_lazy, "to_base_config") + else merged_lazy + ) merged_config_values[field.name] = converted_value else: # No global value to merge with, just convert to base @@ -147,8 +167,11 @@ def _create_merged_config(pipeline_config: 'PipelineConfig', global_config: Glob # This ensures None values in nested configs resolve to global values global_value = getattr(global_config, field.name) from dataclasses import is_dataclass + if is_dataclass(pipeline_value) and is_dataclass(global_value): - merged_config_values[field.name] = _merge_nested_dataclass(pipeline_value, global_value) + merged_config_values[field.name] = _merge_nested_dataclass( + pipeline_value, global_value + ) else: # Regular value - use as-is merged_config_values[field.name] = pipeline_value @@ -163,7 +186,11 @@ def _create_merged_config(pipeline_config: 'PipelineConfig', global_config: Glob def _execute_axis_with_sequential_combinations( pipeline_definition: List[AbstractStep], axis_contexts: List[tuple], # List of (context_key, frozen_context) tuples - visualizer: Optional['NapariVisualizerType'] + visualizer: Optional["NapariVisualizerType"], + execution_id: str, + plate_id: str, + worker_slot: str, + owned_wells: List[str], ) -> ExecutionResult: """ Execute all sequential combinations for a single axis in order. @@ -175,23 +202,48 @@ def _execute_axis_with_sequential_combinations( pipeline_definition: List of pipeline steps to execute axis_contexts: List of (context_key, frozen_context) tuples for this axis visualizer: Optional Napari visualizer (not used in multiprocessing) + execution_id: Unique identifier for this execution + plate_id: Path or identifier for plate being processed Returns: ExecutionResult with status for the entire axis (all combinations) """ # Precondition: axis_contexts must not be empty if not axis_contexts: - raise ValueError("axis_contexts cannot be empty - this indicates a bug in the caller") + raise ValueError( + "axis_contexts cannot be empty - this indicates a bug in the caller" + ) # Extract axis_id from first context first_context_key, first_context = axis_contexts[0] axis_id = first_context.axis_id - + total_steps = len(pipeline_definition) + + emit( + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name="pipeline", + phase=ProgressPhase.AXIS_STARTED, + status=ProgressStatus.STARTED, + completed=0, + total=total_steps, + percent=0.0, + worker_slot=worker_slot, + owned_wells=owned_wells, + ) for combo_idx, (context_key, frozen_context) in enumerate(axis_contexts): - # Execute this combination - result = _execute_single_axis_static(pipeline_definition, frozen_context, visualizer) + result = _execute_single_axis_static( + pipeline_definition, + frozen_context, + visualizer, + execution_id, + plate_id, + worker_slot, + owned_wells, + ) # Clear VFS after each combination to prevent memory accumulation # This must happen REGARDLESS of success/failure to prevent memory leaks @@ -205,20 +257,53 @@ def _execute_axis_with_sequential_combinations( # Check if this combination failed (after cleanup to prevent memory leaks) if not result.is_success(): - logger.error(f"🔄 WORKER: Combination {context_key} failed for axis {axis_id}") + logger.error( + f"🔄 WORKER: Combination {context_key} failed for axis {axis_id}" + ) + emit( + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name="pipeline", + phase=ProgressPhase.AXIS_ERROR, + status=ProgressStatus.ERROR, + completed=0, + total=total_steps, + percent=0.0, + message=result.error_message, + worker_slot=worker_slot, + owned_wells=owned_wells, + ) return ExecutionResult.error( axis_id=axis_id, failed_combination=context_key, - error_message=result.error_message + error_message=result.error_message, ) + emit( + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name="pipeline", + phase=ProgressPhase.AXIS_COMPLETED, + status=ProgressStatus.SUCCESS, + completed=total_steps, + total=total_steps, + percent=100.0, + worker_slot=worker_slot, + owned_wells=owned_wells, + ) return ExecutionResult.success(axis_id=axis_id) def _execute_single_axis_static( pipeline_definition: List[AbstractStep], - frozen_context: 'ProcessingContext', - visualizer: Optional['NapariVisualizerType'] + frozen_context: "ProcessingContext", + visualizer: Optional["NapariVisualizerType"], + execution_id: str, + plate_id: str, + worker_slot: str, + owned_wells: List[str], ) -> ExecutionResult: """ Static version of _execute_single_axis for multiprocessing compatibility. @@ -230,11 +315,14 @@ def _execute_single_axis_static( pipeline_definition: List of pipeline steps to execute frozen_context: Frozen processing context for this axis visualizer: Optional Napari visualizer (not used in multiprocessing) + execution_id: Unique identifier for this execution + plate_id: Path or identifier for plate being processed Returns: ExecutionResult with status for this axis """ axis_id = frozen_context.axis_id + total_steps = len(pipeline_definition) # NUCLEAR VALIDATION if not frozen_context.is_frozen(): @@ -247,40 +335,96 @@ def _execute_single_axis_static( logger.error(error_msg) raise RuntimeError(error_msg) + # Set execution tracking fields on context (these are allowed even when frozen) + # These fields are set at execution time, not compilation time + object.__setattr__(frozen_context, "execution_id", execution_id) + object.__setattr__(frozen_context, "plate_id", plate_id) + object.__setattr__(frozen_context, "worker_slot", worker_slot) + object.__setattr__(frozen_context, "owned_wells", owned_wells) + # Execute each step in the pipeline for step_index, step in enumerate(pipeline_definition): step_name = frozen_context.step_plans[step_index]["step_name"] - # Verify step has process method (should always be true for AbstractStep subclasses) - # This check is acceptable because AbstractStep is an abstract base class - if not hasattr(step, 'process'): - error_msg = f"Step {step_index+1} missing process method for axis {axis_id}" - logger.error(error_msg) - raise RuntimeError(error_msg) + emit( + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name=step_name, + phase=ProgressPhase.STEP_STARTED, + status=ProgressStatus.STARTED, + completed=step_index, + total=total_steps, + percent=(step_index / total_steps) * 100.0, + worker_slot=worker_slot, + owned_wells=owned_wells, + ) # Call process method on step instance step.process(frozen_context, step_index) + emit( + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name=step_name, + phase=ProgressPhase.STEP_COMPLETED, + status=ProgressStatus.SUCCESS, + completed=step_index + 1, + total=total_steps, + percent=((step_index + 1) / total_steps) * 100.0, + worker_slot=worker_slot, + owned_wells=owned_wells, + ) + # Handle visualization if requested if visualizer: step_plan = frozen_context.step_plans[step_index] - if step_plan['visualize']: - output_dir = step_plan['output_dir'] - write_backend = step_plan['write_backend'] + if step_plan["visualize"]: + output_dir = step_plan["output_dir"] + write_backend = step_plan["write_backend"] if output_dir: - logger.debug(f"Visualizing output for step {step_index} from path {output_dir} (backend: {write_backend}) for axis {axis_id}") + logger.debug( + f"Visualizing output for step {step_index} from path {output_dir} (backend: {write_backend}) for axis {axis_id}" + ) visualizer.visualize_path( step_id=f"step_{step_index}", path=str(output_dir), backend=write_backend, - axis_id=axis_id + axis_id=axis_id, ) else: - logger.warning(f"Step {step_index} in axis {axis_id} flagged for visualization but 'output_dir' is missing in its plan.") + logger.warning( + f"Step {step_index} in axis {axis_id} flagged for visualization but 'output_dir' is missing in its plan." + ) return ExecutionResult.success(axis_id=axis_id) +def _execute_worker_lane_static( + pipeline_definition: List[AbstractStep], + lane_axis_contexts: List[tuple[str, List[tuple]]], + visualizer: Optional["NapariVisualizerType"], + execution_id: str, + plate_id: str, + worker_slot: str, + owned_wells: List[str], +) -> Dict[str, ExecutionResult]: + """Execute a deterministic worker lane: wells sequentially within one slot.""" + lane_results: Dict[str, ExecutionResult] = {} + for axis_id, axis_contexts in lane_axis_contexts: + lane_results[axis_id] = _execute_axis_with_sequential_combinations( + pipeline_definition=pipeline_definition, + axis_contexts=axis_contexts, + visualizer=visualizer, + execution_id=execution_id, + plate_id=plate_id, + worker_slot=worker_slot, + owned_wells=owned_wells, + ) + return lane_results + + def _configure_worker_logging(log_file_base: str): """ Configure logging and import hook for worker process. @@ -303,7 +447,9 @@ def _configure_worker_logging(log_file_base: str): # Create unique worker identifier using PID and timestamp worker_pid = os.getpid() - worker_timestamp = int(time.time() * 1000000) # Microsecond precision for uniqueness + worker_timestamp = int( + time.time() * 1000000 + ) # Microsecond precision for uniqueness worker_id = f"{worker_pid}_{worker_timestamp}" worker_log_file = f"{log_file_base}_worker_{worker_id}.log" @@ -313,7 +459,9 @@ def _configure_worker_logging(log_file_base: str): # Create file handler for worker logs file_handler = logging.FileHandler(worker_log_file) - file_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) + file_handler.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ) root_logger.addHandler(file_handler) root_logger.setLevel(logging.INFO) @@ -324,7 +472,12 @@ def _configure_worker_logging(log_file_base: str): worker_logger = logging.getLogger("openhcs.worker") -def _configure_worker_with_gpu(log_file_base: str, global_config_dict: dict): +def _configure_worker_with_gpu( + log_file_base: str, + global_config_dict: dict, + progress_queue=None, + progress_context=None, +): """ Configure logging, function registry, and GPU registry for worker process. @@ -341,8 +494,8 @@ def _configure_worker_with_gpu(log_file_base: str, global_config_dict: dict): # Workers should be allowed to import GPU libs if available. # The parent subprocess runner may set OPENHCS_SUBPROCESS_NO_GPU=1 to stay lean, # but that flag must not leak into worker processes. - os.environ.pop('OPENHCS_SUBPROCESS_NO_GPU', None) - os.environ.pop('POLYSTORE_SUBPROCESS_NO_GPU', None) + os.environ.pop("OPENHCS_SUBPROCESS_NO_GPU", None) + os.environ.pop("POLYSTORE_SUBPROCESS_NO_GPU", None) # Configure logging only if log_file_base is provided if log_file_base: @@ -371,22 +524,27 @@ def _configure_worker_with_gpu(log_file_base: str, global_config_dict: dict): try: # Reconstruct global config from dict from openhcs.core.config import GlobalPipelineConfig + global_config = GlobalPipelineConfig(**global_config_dict) # Initialize GPU registry for this worker from openhcs.core.orchestrator.gpu_scheduler import setup_global_gpu_registry + setup_global_gpu_registry(global_config) except Exception as e: # Don't raise - let worker continue without GPU if needed pass + if progress_queue is not None and progress_context is not None: + from openhcs.core.progress import set_progress_queue -# Global variable to store log file base for worker processes -_worker_log_file_base = None - + # Set progress queue for worker processes + set_progress_queue(progress_queue) +# Global variable to store log file base for worker processes +_worker_log_file_base = None class PipelineOrchestrator: @@ -405,16 +563,16 @@ class PipelineOrchestrator: # ObjectState delegation: when ObjectState stores this orchestrator, extract # editable parameters from pipeline_config (a dataclass) instead of the orchestrator. # This enables time-travel to track the orchestrator lifecycle while forms edit the config. - __objectstate_delegate__ = 'pipeline_config' + __objectstate_delegate__ = "pipeline_config" def __init__( self, plate_path: Union[str, Path], workspace_path: Optional[Union[str, Path]] = None, *, - pipeline_config: Optional['PipelineConfig'] = None, + pipeline_config: Optional["PipelineConfig"] = None, storage_registry: Optional[Any] = None, - progress_callback: Optional[Callable[[str, str, str, Dict[str, Any]], None]] = None, + progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None, ): # Lock removed - was orphaned code never used @@ -428,6 +586,7 @@ def __init__( # Track executor for cancellation support self._executor = None self._cancelled = False # Track cancellation requests + self.execution_id = f"local::{plate_path}" # Initialize auto-sync control for pipeline config self._pipeline_config = None @@ -440,6 +599,7 @@ def __init__( # This ensures the orchestrator has a dataclass attribute for stack introspection # PipelineConfig is already the lazy version of GlobalPipelineConfig from openhcs.core.config import PipelineConfig + if pipeline_config is None: # CRITICAL FIX: Create pipeline config that inherits from global config # This ensures the orchestrator's pipeline_config has the global values for resolution @@ -455,7 +615,9 @@ def __init__( # The resolver's _is_context_provider method only finds public attributes (skips _private) # This allows the resolver to discover the orchestrator's pipeline config during context resolution self.pipeline_config = pipeline_config - logger.info("PipelineOrchestrator initialized with PipelineConfig for context discovery.") + logger.info( + "PipelineOrchestrator initialized with PipelineConfig for context discovery." + ) # REMOVED: Unnecessary thread-local modification # The orchestrator should not modify thread-local storage during initialization @@ -472,19 +634,23 @@ def __init__( if not plate_path.exists(): raise FileNotFoundError(f"Plate path does not exist: {plate_path}") if not plate_path.is_dir(): - raise NotADirectoryError(f"Plate path is not a directory: {plate_path}") + raise NotADirectoryError( + f"Plate path is not a directory: {plate_path}" + ) # Initialize _plate_path_frozen first to allow plate_path to be set during initialization - object.__setattr__(self, '_plate_path_frozen', False) + object.__setattr__(self, "_plate_path_frozen", False) self.plate_path = plate_path self.workspace_path = workspace_path if self.plate_path is None and self.workspace_path is None: - raise ValueError("Either plate_path or workspace_path must be provided for PipelineOrchestrator.") + raise ValueError( + "Either plate_path or workspace_path must be provided for PipelineOrchestrator." + ) # Freeze plate_path immediately after setting it to prove immutability - object.__setattr__(self, '_plate_path_frozen', True) + object.__setattr__(self, "_plate_path_frozen", True) logger.info(f"🔒 PLATE_PATH FROZEN: {self.plate_path} is now immutable") if storage_registry: @@ -493,7 +659,11 @@ def __init__( else: # Use the global registry directly (don't copy) so that reset_memory_backend() works correctly # The global registry is a singleton, and VFS clearing needs to clear the same instance - from polystore.base import storage_registry as global_storage_registry, ensure_storage_registry + from polystore.base import ( + storage_registry as global_storage_registry, + ensure_storage_registry, + ) + # Ensure registry is initialized ensure_storage_registry() self.registry = global_storage_registry @@ -503,7 +673,9 @@ def __init__( shared_context = get_current_global_config(GlobalPipelineConfig) zarr_backend_with_config = ZarrStorageBackend(shared_context.zarr_config) self.registry[Backend.ZARR.value] = zarr_backend_with_config - logger.info(f"Orchestrator zarr backend configured with {shared_context.zarr_config.compressor.value} compression") + logger.info( + f"Orchestrator zarr backend configured with {shared_context.zarr_config.compressor.value} compression" + ) # Orchestrator always creates its own FileManager, using the determined registry self.filemanager = FileManager(self.registry) @@ -519,25 +691,26 @@ def __init__( logger.info("PipelineOrchestrator initialized with progress callback") # Component keys cache for fast access - uses AllComponents (includes multiprocessing axis) - self._component_keys_cache: Dict['AllComponents', List[str]] = {} + self._component_keys_cache: Dict["AllComponents", List[str]] = {} # Metadata cache service - per-orchestrator instance (not global singleton) from openhcs.core.metadata_cache import MetadataCache + self._metadata_cache_service = MetadataCache() # Viewer management - shared between pipeline execution and image browser self._visualizers = {} # Dict[(backend_name, port)] -> visualizer instance - def __setattr__(self, name: str, value: Any) -> None: """ Set an attribute, preventing modification of plate_path after it's frozen. This proves that plate_path is truly immutable after initialization. """ - if name == 'plate_path' and getattr(self, '_plate_path_frozen', False): + if name == "plate_path" and getattr(self, "_plate_path_frozen", False): import traceback - stack_trace = ''.join(traceback.format_stack()) + + stack_trace = "".join(traceback.format_stack()) error_msg = ( f"🚫 IMMUTABLE PLATE_PATH VIOLATION: Cannot modify plate_path after freezing!\n" f"Current value: {getattr(self, 'plate_path', 'UNSET')}\n" @@ -569,49 +742,45 @@ def get_or_create_visualizer(self, config, vis_config=None): """ from openhcs.core.config import StreamingConfig - # Generic streaming config handling using polymorphic attributes + # Streaming configs should be managed by the centralized ViewerStateManager if isinstance(config, StreamingConfig): - # Pre-create queue tracker using polymorphic attributes + # Ensure a queue tracker exists for this viewer from zmqruntime.queue_tracker import GlobalQueueTrackerRegistry + registry = GlobalQueueTrackerRegistry() registry.get_or_create_tracker(config.port, config.viewer_type) key = (config.viewer_type, config.port) - else: - backend_name = config.backend.name if hasattr(config, 'backend') else 'unknown' - key = (backend_name,) - - # Check if we already have a visualizer for this key - if key in self._visualizers: - vis = self._visualizers[key] - if vis.is_running: - return vis - else: - del self._visualizers[key] - # Create new visualizer using polymorphic create_visualizer method - vis = config.create_visualizer(self.filemanager, vis_config) + # Use zmqruntime's atomic get_or_create to avoid races that spawned duplicate viewers + from zmqruntime import get_or_create_viewer - # Start viewer asynchronously for streaming configs - if isinstance(config, StreamingConfig): - vis.start_viewer(async_mode=True) + viewer, created = get_or_create_viewer( + viewer_type=config.viewer_type, + port=config.port, + factory=lambda: config.create_visualizer(self.filemanager, vis_config), + wait_for_ready=True, + ready_timeout=30.0, + ) - # Ping server to set ready state (background thread to avoid blocking) - import threading - def ping_server(): - import time - time.sleep(1.0) # Give server time to start - if hasattr(vis, '_wait_for_server_ready'): - vis._wait_for_server_ready(timeout=10.0) + # Keep a reference for backward compatibility + self._visualizers[key] = viewer + return viewer - thread = threading.Thread(target=ping_server, daemon=True) - thread.start() - else: + # Non-streaming (local) visualizers: create and start synchronously + vis = config.create_visualizer(self.filemanager, vis_config) + try: vis.start_viewer() + except Exception: + # Some visualizers expose start() instead of start_viewer() + if hasattr(vis, "start"): + vis.start() + else: + logger.exception("Failed to start non-streaming visualizer") - # Store in cache - self._visualizers[key] = vis - + # Store for compatibility + backend_name = config.backend.name if hasattr(config, "backend") else "unknown" + self._visualizers[(backend_name,)] = vis return vis def initialize_microscope_handler(self): @@ -619,27 +788,39 @@ def initialize_microscope_handler(self): if self.microscope_handler is not None: logger.debug("Microscope handler already initialized.") return -# if self.input_dir is None: -# raise RuntimeError("Workspace (and input_dir) must be initialized before microscope handler.") + # if self.input_dir is None: + # raise RuntimeError("Workspace (and input_dir) must be initialized before microscope handler.") - logger.info(f"Initializing microscope handler using input directory: {self.input_dir}...") + logger.info( + f"Initializing microscope handler using input directory: {self.input_dir}..." + ) try: # Use configured microscope type or auto-detect # Use SAVED global config (not live edits) for orchestrator initialization - shared_context = get_current_global_config(GlobalPipelineConfig, use_live=False) - microscope_type = shared_context.microscope.value if shared_context.microscope != Microscope.AUTO else 'auto' + shared_context = get_current_global_config( + GlobalPipelineConfig, use_live=False + ) + microscope_type = ( + shared_context.microscope.value + if shared_context.microscope != Microscope.AUTO + else "auto" + ) self.microscope_handler = create_microscope_handler( plate_folder=str(self.plate_path), filemanager=self.filemanager, microscope_type=microscope_type, ) - logger.info(f"Initialized microscope handler: {type(self.microscope_handler).__name__}") + logger.info( + f"Initialized microscope handler: {type(self.microscope_handler).__name__}" + ) except Exception as e: error_msg = f"Failed to create microscope handler: {e}" logger.error(error_msg) raise RuntimeError(error_msg) from e - def initialize(self, workspace_path: Optional[Union[str, Path]] = None) -> 'PipelineOrchestrator': + def initialize( + self, workspace_path: Optional[Union[str, Path]] = None + ) -> "PipelineOrchestrator": """ Initializes all required components for the orchestrator. Must be called before other processing methods. @@ -663,10 +844,31 @@ def initialize(self, workspace_path: Optional[Union[str, Path]] = None) -> 'Pipe self.input_dir = Path(actual_image_dir) logger.info(f"Set input directory to: {self.input_dir}") + # Log effective backend intent early for debugging test/UI differences + try: + vfs_cfg = ( + self.get_effective_config().vfs_config + if self.pipeline_config + else None + ) + if vfs_cfg is not None: + logger.info( + "VFS config at init: read_backend=%s intermediate_backend=%s materialization_backend=%s", + getattr(vfs_cfg, "read_backend", None), + getattr(vfs_cfg, "intermediate_backend", None), + getattr(vfs_cfg, "materialization_backend", None), + ) + except Exception: + logger.debug("Could not log VFS config at init", exc_info=True) + # Set workspace_path based on what the handler returned if actual_image_dir != self.plate_path: # Handler created a workspace (or virtual path for OMERO) - self.workspace_path = Path(actual_image_dir).parent if Path(actual_image_dir).name != "workspace" else Path(actual_image_dir) + self.workspace_path = ( + Path(actual_image_dir).parent + if Path(actual_image_dir).name != "workspace" + else Path(actual_image_dir) + ) else: # Handler used plate directly (like OpenHCS) self.workspace_path = None @@ -679,15 +881,15 @@ def initialize(self, workspace_path: Optional[Union[str, Path]] = None) -> 'Pipe logger.info("Caching component keys and metadata...") self.cache_component_keys() self._metadata_cache_service.cache_metadata( - self.microscope_handler, - self.plate_path, - self._component_keys_cache + self.microscope_handler, self.plate_path, self._component_keys_cache ) # Ensure complete OpenHCS metadata exists self._ensure_openhcs_metadata() - logger.info("PipelineOrchestrator fully initialized with cached component keys and metadata.") + logger.info( + "PipelineOrchestrator fully initialized with cached component keys and metadata." + ) return self except Exception as e: self._state = OrchestratorState.INIT_FAILED @@ -710,20 +912,25 @@ def _ensure_openhcs_metadata(self) -> None: # Skip metadata creation for OMERO and other non-disk-based handlers # OMERO uses virtual paths like /omero/plate_1 which are not real directories - if self.microscope_handler.microscope_type == 'omero': - logger.debug("Skipping metadata creation for OMERO plate (uses virtual paths)") + if self.microscope_handler.microscope_type == "omero": + logger.debug( + "Skipping metadata creation for OMERO plate (uses virtual paths)" + ) return # For plates with virtual workspace, metadata is already created by _build_virtual_mapping() # We just need to add the component metadata to the existing "." subdirectory from polystore.metadata_writer import get_subdirectory_name + subdir_name = get_subdirectory_name(self.input_dir, self.plate_path) # Create context using SAME logic as create_context() to get full metadata context = self.create_context(axis_id="metadata_init") # Determine correct backend using handler's logic (virtual_workspace for ImageXpress/Opera, disk for others) - backend = self.microscope_handler.get_primary_backend(self.plate_path, self.filemanager) + backend = self.microscope_handler.get_primary_backend( + self.plate_path, self.filemanager + ) logger.debug(f"Using backend '{backend}' for metadata extraction") # Create metadata (will skip if already complete) @@ -735,7 +942,7 @@ def _ensure_openhcs_metadata(self) -> None: is_main=True, plate_root=str(self.plate_path), sub_dir=subdir_name, - skip_if_complete=True + skip_if_complete=True, ) def get_results_path(self) -> Path: @@ -760,9 +967,7 @@ def get_results_path(self) -> Path: # Use path_planning_config from global config path_config = self.global_config.path_planning_config output_plate_root = PipelinePathPlanner.build_output_plate_root( - self.plate_path, - path_config, - is_per_step_materialization=False + self.plate_path, path_config, is_per_step_materialization=False ) return output_plate_root / materialization_path @@ -770,16 +975,20 @@ def get_results_path(self) -> Path: def create_context(self, axis_id: str) -> ProcessingContext: """Creates a ProcessingContext for a given multiprocessing axis value.""" if not self.is_initialized(): - raise RuntimeError("Orchestrator must be initialized before calling create_context().") + raise RuntimeError( + "Orchestrator must be initialized before calling create_context()." + ) if not axis_id: raise ValueError("Axis identifier must be provided.") if self.input_dir is None: - raise RuntimeError("Orchestrator input_dir is not set; initialize orchestrator first.") + raise RuntimeError( + "Orchestrator input_dir is not set; initialize orchestrator first." + ) context = ProcessingContext( global_config=self.get_effective_config(), axis_id=axis_id, - filemanager=self.filemanager + filemanager=self.filemanager, ) # Orchestrator reference removed - was orphaned and unpickleable context.microscope_handler = self.microscope_handler @@ -791,7 +1000,9 @@ def create_context(self, axis_id: str) -> ProcessingContext: # Extract cached metadata from service and convert to dict format expected by OpenHCSMetadataGenerator metadata_dict = {} for component in AllComponents: - cached_metadata = self._metadata_cache_service.get_cached_metadata(component) + cached_metadata = self._metadata_cache_service.get_cached_metadata( + component + ) if cached_metadata: metadata_dict[component] = cached_metadata context.metadata_cache = metadata_dict @@ -802,171 +1013,18 @@ def compile_pipelines( self, pipeline_definition: List[AbstractStep], well_filter: Optional[List[str]] = None, - enable_visualizer_override: bool = False + enable_visualizer_override: bool = False, + is_zmq_execution: bool = False, ) -> Dict[str, ProcessingContext]: """Compile pipelines for axis values (well_filter name preserved for UI compatibility).""" return PipelineCompiler.compile_pipelines( orchestrator=self, pipeline_definition=pipeline_definition, axis_filter=well_filter, # Translate well_filter to axis_filter for generic backend - enable_visualizer_override=enable_visualizer_override + enable_visualizer_override=enable_visualizer_override, + is_zmq_execution=is_zmq_execution, ) - def _execute_single_axis( - self, - pipeline_definition: List[AbstractStep], - frozen_context: ProcessingContext, - visualizer: Optional[NapariVisualizerType] - ) -> Dict[str, Any]: - """Executes the pipeline for a single well using its frozen context.""" - axis_id = frozen_context.axis_id - - # Send progress: axis started - if self.progress_callback: - try: - self.progress_callback(axis_id, 'pipeline', 'started', { - 'total_steps': len(pipeline_definition) - }) - except Exception as e: - logger.warning(f"Progress_callback failed for axis {axis_id} start: {e}") - - # NUCLEAR VALIDATION - if not frozen_context.is_frozen(): - error_msg = f"🔥 SINGLE_AXIS ERROR: Context for axis {axis_id} is not frozen before execution" - logger.error(error_msg) - raise RuntimeError(error_msg) - - if not pipeline_definition: - error_msg = f"🔥 SINGLE_AXIS ERROR: Empty pipeline_definition for axis {axis_id}" - logger.error(error_msg) - raise RuntimeError(error_msg) - - - # All execution goes through step-sequential now - # Sequential combinations are handled by compiling separate contexts - return self._execute_step_sequential(pipeline_definition, frozen_context, visualizer) - - def _execute_step_sequential( - self, - pipeline_definition: List[AbstractStep], - frozen_context: ProcessingContext, - visualizer: Optional[NapariVisualizerType] - ) -> Dict[str, Any]: - """Execute pipeline with step-wide sequential processing (current behavior).""" - axis_id = frozen_context.axis_id - - for step_index, step in enumerate(pipeline_definition): - step_name = frozen_context.step_plans[step_index]["step_name"] - - # Send progress: step started - if self.progress_callback: - try: - self.progress_callback(axis_id, step_name, 'started', { - 'step_index': step_index, - 'total_steps': len(pipeline_definition) - }) - except Exception as e: - logger.warning(f"Progress callback failed for axis {axis_id} step {step_name} start: {e}") - - # Verify step has process method - if not hasattr(step, 'process'): - error_msg = f"🔥 SINGLE_AXIS ERROR: Step {step_index+1} missing process method for axis {axis_id}" - logger.error(error_msg) - raise RuntimeError(error_msg) - - # Call process method on step instance - step.process(frozen_context, step_index) - - # Send progress: step completed - if self.progress_callback: - try: - self.progress_callback(axis_id, step_name, 'completed', { - 'step_index': step_index, - 'total_steps': len(pipeline_definition) - }) - except Exception as e: - logger.warning(f"Progress callback failed for axis {axis_id} step {step_name} completion: {e}") - - if visualizer: - step_plan = frozen_context.step_plans[step_index] - if step_plan['visualize']: - output_dir = step_plan['output_dir'] - write_backend = step_plan['write_backend'] - if output_dir: - logger.debug(f"Visualizing output for step {step_index} from path {output_dir} (backend: {write_backend}) for axis {axis_id}") - visualizer.visualize_path( - step_id=f"step_{step_index}", - path=str(output_dir), - backend=write_backend, - axis_id=axis_id - ) - else: - logger.warning(f"Step {step_index} in axis {axis_id} flagged for visualization but 'output_dir' is missing in its plan.") - - - # Send progress: axis completed - if self.progress_callback: - try: - self.progress_callback(axis_id, 'pipeline', 'completed', { - 'total_steps': len(pipeline_definition) - }) - except Exception as e: - logger.warning(f"Progress callback failed for axis {axis_id} completion: {e}") - - return {"status": "success", "axis_id": axis_id} - - def _execute_pipeline_sequential( - self, - pipeline_definition: List[AbstractStep], - frozen_context: ProcessingContext, - visualizer: Optional[NapariVisualizerType] - ) -> Dict[str, Any]: - """Execute pipeline with pipeline-wide sequential processing. - - Combinations are precomputed at compile time and stored in context.pipeline_sequential_combinations. - Loop: combinations → steps (process one combo through all steps before moving to next). - VFS is cleared between combinations to prevent memory accumulation. - """ - axis_id = frozen_context.axis_id - - # Combinations were precomputed at compile time - combinations = frozen_context.pipeline_sequential_combinations or [] - - if not combinations: - logger.warning(f"Pipeline sequential mode enabled but no combinations found for axis {axis_id}") - # Fallback to normal execution - for step_index, step in enumerate(pipeline_definition): - step.process(frozen_context, step_index) - else: - - # Loop: combinations → steps - for combo_idx, combo in enumerate(combinations): - frozen_context.current_sequential_combination = combo - - # Process all steps for this combination - for step_index, step in enumerate(pipeline_definition): - step.process(frozen_context, step_index) - - # Clear VFS after each combination to prevent memory accumulation - try: - from polystore.base import reset_memory_backend - from openhcs.core.memory import cleanup_all_gpu_frameworks - - reset_memory_backend() - cleanup_all_gpu_frameworks() - except Exception as e: - logger.warning(f"Failed to clear VFS after combination {combo}: {e}") - - frozen_context.current_sequential_combination = None - - if self.progress_callback: - try: - self.progress_callback(axis_id, 'pipeline', 'completed', {'total_steps': len(pipeline_definition)}) - except Exception as e: - logger.warning(f"Progress callback failed: {e}") - - return {"status": "success", "axis_id": axis_id} - def cancel_execution(self): """ Cancel ongoing execution by shutting down the executor. @@ -989,7 +1047,10 @@ def execute_compiled_plate( compiled_contexts: Dict[str, ProcessingContext], max_workers: Optional[int] = None, visualizer: Optional[NapariVisualizerType] = None, - log_file_base: Optional[str] = None + log_file_base: Optional[str] = None, + progress_queue=None, + progress_context=None, + worker_assignments: Optional[Dict[str, List[str]]] = None, ) -> Dict[str, Dict[str, Any]]: """ Execute-all phase: Runs the stateless pipeline against compiled contexts. @@ -1010,21 +1071,33 @@ def execute_compiled_plate( # CRITICAL FIX: Use resolved pipeline definition from compilation if available # For subprocess runner, use the parameter directly since it receives pre-compiled contexts - resolved_pipeline = getattr(self, '_resolved_pipeline_definition', None) + resolved_pipeline = self.__dict__.get("_resolved_pipeline_definition") if resolved_pipeline is not None: pipeline_definition = resolved_pipeline if not self.is_initialized(): - raise RuntimeError("Orchestrator must be initialized before executing.") + raise RuntimeError("Orchestrator must be initialized before executing.") if not pipeline_definition: - raise ValueError("A valid (stateless) pipeline definition must be provided.") + raise ValueError( + "A valid (stateless) pipeline definition must be provided." + ) if not compiled_contexts: logger.warning("No compiled contexts provided for execution.") return {} - + if progress_queue is None: + raise ValueError( + "progress_queue is required for execute_compiled_plate invariant path" + ) + if progress_context is None: + raise ValueError( + "progress_context is required for execute_compiled_plate invariant path" + ) + execution_id = progress_context["execution_id"] + plate_id = progress_context["plate_id"] + # Access num_workers from effective config (merged pipeline + global config) actual_max_workers = max_workers or self.get_effective_config().num_workers - if actual_max_workers <= 0: # Ensure positive number of workers + if actual_max_workers <= 0: # Ensure positive number of workers actual_max_workers = 1 # 🔬 AUTOMATIC VISUALIZER CREATION: Create visualizers if compiler detected streaming @@ -1036,19 +1109,39 @@ def execute_compiled_plate( unique_configs = {} for ctx in compiled_contexts.values(): for visualizer_info in ctx.required_visualizers: - config = visualizer_info['config'] - key = (config.viewer_type, config.port) if isinstance(config, StreamingConfig) else (config.backend.name,) + config = visualizer_info["config"] + key = ( + (config.viewer_type, config.port) + if isinstance(config, StreamingConfig) + else (config.backend.name,) + ) if key not in unique_configs: unique_configs[key] = (config, ctx.visualizer_config) - # Create visualizers - for config, vis_config in unique_configs.values(): + # Create visualizers — emit progress for each launch + for key, (config, vis_config) in unique_configs.items(): + viewer_name = key[0] if key else type(config).__name__ + viewer_port = key[1] if len(key) > 1 else None + port_info = f" on port {viewer_port}" if viewer_port else "" + progress_queue.put( + create_event( + execution_id=execution_id, + plate_id=plate_id, + axis_id="", + step_name="", + phase=ProgressPhase.INIT, + status=ProgressStatus.STARTED, + percent=0.0, + message=f"Launching {viewer_name} viewer{port_info}", + ).to_dict() + ) visualizers.append(self.get_or_create_visualizer(config, vis_config)) # Wait for all streaming viewers to be ready before starting pipeline # This ensures viewers are available to receive images if visualizers: import time + max_wait = 30.0 # Maximum wait time in seconds start_time = time.time() @@ -1059,51 +1152,88 @@ def execute_compiled_plate( all_ready = all(v.is_running for v in visualizers) if all_ready: + progress_queue.put( + create_event( + execution_id=execution_id, + plate_id=plate_id, + axis_id="", + step_name="", + phase=ProgressPhase.INIT, + status=ProgressStatus.RUNNING, + percent=0.0, + message="All streaming viewers ready", + ).to_dict() + ) break time.sleep(0.2) # Check every 200ms else: # Timeout - log which viewers aren't ready (use generic port attribute) not_ready = [v.port for v in visualizers if not v.is_running] - logger.warning(f"🔬 ORCHESTRATOR: Timeout waiting for streaming viewers. Not ready: {not_ready}") + logger.warning( + f"🔬 ORCHESTRATOR: Timeout waiting for streaming viewers. Not ready: {not_ready}" + ) + progress_queue.put( + create_event( + execution_id=execution_id, + plate_id=plate_id, + axis_id="", + step_name="", + phase=ProgressPhase.INIT, + status=ProgressStatus.RUNNING, + percent=0.0, + message=f"Timeout waiting for streaming viewers. Not ready: {not_ready}", + ).to_dict() + ) # Clear viewer state for new pipeline run to prevent accumulation for vis in visualizers: - if hasattr(vis, 'clear_viewer_state'): + if hasattr(vis, "clear_viewer_state"): success = vis.clear_viewer_state() if not success: - logger.warning(f"🔬 ORCHESTRATOR: Failed to clear state for viewer on port {vis.port}") + logger.warning( + f"🔬 ORCHESTRATOR: Failed to clear state for viewer on port {vis.port}" + ) # For backwards compatibility, set visualizer to the first one visualizer = visualizers[0] if visualizers else None - # Reset cancellation flag at start of execution - self._cancelled = False + set_progress_queue(progress_queue) + try: + # Reset cancellation flag at start of execution + self._cancelled = False - self._state = OrchestratorState.EXECUTING - logger.info(f"Starting execution for {len(compiled_contexts)} axis values with max_workers={actual_max_workers}.") + self._state = OrchestratorState.EXECUTING + logger.info( + f"Starting execution for {len(compiled_contexts)} axis values with max_workers={actual_max_workers}." + ) - try: execution_results: Dict[str, ExecutionResult] = {} # CUDA COMPATIBILITY: Set spawn method for multiprocessing to support CUDA try: # Check if spawn method is available and set it if not already set current_method = multiprocessing.get_start_method(allow_none=True) - if current_method != 'spawn': - multiprocessing.set_start_method('spawn', force=True) + if current_method != "spawn": + multiprocessing.set_start_method("spawn", force=True) except RuntimeError as e: # Start method may already be set, which is fine pass # Choose executor type based on effective config for debugging support effective_config = self.get_effective_config() - executor_type = "ThreadPoolExecutor" if effective_config.use_threading else "ProcessPoolExecutor" + executor_type = ( + "ThreadPoolExecutor" + if effective_config.use_threading + else "ProcessPoolExecutor" + ) # DEATH DETECTION: Mark executor creation # Choose appropriate executor class and configure worker logging if effective_config.use_threading: - executor = concurrent.futures.ThreadPoolExecutor(max_workers=actual_max_workers) + executor = concurrent.futures.ThreadPoolExecutor( + max_workers=actual_max_workers + ) else: # CRITICAL FIX: Use _configure_worker_with_gpu to ensure workers have function registry # Workers need the function registry to access decorated functions with memory types @@ -1114,13 +1244,23 @@ def execute_compiled_plate( executor = concurrent.futures.ProcessPoolExecutor( max_workers=actual_max_workers, initializer=_configure_worker_with_gpu, - initargs=(log_file_base, global_config_dict) + initargs=( + log_file_base, + global_config_dict, + progress_queue, + progress_context, + ), ) else: executor = concurrent.futures.ProcessPoolExecutor( max_workers=actual_max_workers, initializer=_configure_worker_with_gpu, - initargs=("", global_config_dict) # Empty string for no logging + initargs=( + "", + global_config_dict, + progress_queue, + progress_context, + ), # Empty string for no logging ) # Store executor for cancellation support @@ -1130,23 +1270,27 @@ def execute_compiled_plate( # This can happen if workers are killed externally (e.g., during cancellation) try: with executor: - # NUCLEAR ERROR TRACING: Create snapshot of compiled_contexts to prevent iteration issues contexts_snapshot = dict(compiled_contexts.items()) # CRITICAL FIX: Resolve all lazy dataclass instances before multiprocessing # This ensures that the contexts are safe for pickling in ProcessPoolExecutor # Note: Don't resolve pipeline_definition as it may overwrite collision-resolved configs - contexts_snapshot = resolve_lazy_configurations_for_serialization(contexts_snapshot) + contexts_snapshot = resolve_lazy_configurations_for_serialization( + contexts_snapshot + ) - future_to_axis_id = {} + future_to_worker_slot = {} config = get_openhcs_config() if not config: - raise RuntimeError("Component configuration is required for orchestrator execution") + raise RuntimeError( + "Component configuration is required for orchestrator execution" + ) axis_name = config.multiprocessing_axis.value # Group contexts by axis to detect sequential combinations from collections import defaultdict + contexts_by_axis = defaultdict(list) for context_key, context in contexts_snapshot.items(): # Extract axis_id from context_key (either "axis_id" or "axis_id__combo_N") @@ -1156,79 +1300,149 @@ def execute_compiled_plate( else: contexts_by_axis[context_key].append((context_key, context)) + if worker_assignments is None: + generated: Dict[str, List[str]] = { + f"worker_{idx}": [] for idx in range(actual_max_workers) + } + for idx, axis_id in enumerate(sorted(contexts_by_axis.keys())): + generated[f"worker_{idx % actual_max_workers}"].append( + axis_id + ) + worker_assignments = { + worker_slot: owned + for worker_slot, owned in generated.items() + if owned + } + + # Validate worker ownership map against compiled axis ids. + expected_axis_ids = set(contexts_by_axis.keys()) + all_assigned_axis_ids: list[str] = [] + for owned in worker_assignments.values(): + all_assigned_axis_ids.extend(owned) + if len(all_assigned_axis_ids) != len(set(all_assigned_axis_ids)): + raise RuntimeError( + f"Duplicate axis ownership detected in worker_assignments: {worker_assignments}" + ) + if set(all_assigned_axis_ids) != expected_axis_ids: + raise RuntimeError( + f"worker_assignments mismatch. expected={sorted(expected_axis_ids)}, got={sorted(all_assigned_axis_ids)}" + ) + + axis_to_worker: Dict[str, str] = {} + for worker_slot, owned in worker_assignments.items(): + for axis_id in owned: + axis_to_worker[axis_id] = worker_slot - # Submit one task per axis (each task handles its own sequential combinations) - for axis_id, axis_contexts in contexts_by_axis.items(): + lane_axis_contexts: Dict[str, List[tuple[str, List[tuple]]]] = { + worker_slot: [] for worker_slot in worker_assignments.keys() + } + for axis_id, axis_contexts in contexts_by_axis.items(): try: # Resolve all contexts for this axis resolved_axis_contexts = [ - (context_key, resolve_lazy_configurations_for_serialization(context)) + ( + context_key, + resolve_lazy_configurations_for_serialization( + context + ), + ) for context_key, context in axis_contexts ] - - # Submit task that will handle all combinations for this axis sequentially - future = executor.submit( - _execute_axis_with_sequential_combinations, - pipeline_definition, - resolved_axis_contexts, - None # visualizer + worker_slot = axis_to_worker[axis_id] + lane_axis_contexts[worker_slot].append( + (axis_id, resolved_axis_contexts) ) - future_to_axis_id[future] = axis_id except Exception as submit_error: error_msg = f"🔥 ORCHESTRATOR ERROR: Failed to submit task for {axis_name} {axis_id}: {submit_error}" logger.error(error_msg, exc_info=True) # FAIL-FAST: Re-raise task submission errors immediately raise + for worker_slot, lane_contexts in lane_axis_contexts.items(): + if not lane_contexts: + continue + owned_wells = list(worker_assignments[worker_slot]) + try: + future = executor.submit( + _execute_worker_lane_static, + pipeline_definition, + lane_contexts, + None, # visualizer + execution_id, + plate_id, + worker_slot, + owned_wells, + ) + future_to_worker_slot[future] = (worker_slot, owned_wells) + except Exception as submit_error: + error_msg = f"🔥 ORCHESTRATOR ERROR: Failed to submit lane {worker_slot}: {submit_error}" + logger.error(error_msg, exc_info=True) + raise - completed_count = 0 - for future in concurrent.futures.as_completed(future_to_axis_id): - axis_id = future_to_axis_id[future] - completed_count += 1 + for future in concurrent.futures.as_completed( + future_to_worker_slot + ): + worker_slot, owned_wells = future_to_worker_slot[future] try: - result = future.result() - execution_results[axis_id] = result + lane_results = future.result() + execution_results.update(lane_results) except Exception as exc: import traceback + full_traceback = traceback.format_exc() - error_msg = f"{axis_name.title()} {axis_id} generated an exception during execution: {exc}" - logger.error(f"🔥 ORCHESTRATOR ERROR: {error_msg}", exc_info=True) - logger.error(f"🔥 ORCHESTRATOR FULL TRACEBACK for {axis_name} {axis_id}:\n{full_traceback}") - - # Send error to UI immediately via progress callback - if self.progress_callback: - try: - self.progress_callback(axis_id, 'pipeline', 'error', { - 'error_message': str(exc), - 'traceback': full_traceback - }) - except Exception as cb_error: - logger.warning(f"Progress callback failed for error reporting: {cb_error}") + error_msg = f"Worker lane {worker_slot} generated an exception during execution: {exc}" + logger.error( + f"🔥 ORCHESTRATOR ERROR: {error_msg}", exc_info=True + ) + logger.error( + f"🔥 ORCHESTRATOR FULL TRACEBACK for worker lane {worker_slot}:\n{full_traceback}" + ) + failing_axis_id = owned_wells[0] if owned_wells else "" + + emit( + execution_id=execution_id, + plate_id=plate_id, + axis_id=failing_axis_id, + step_name="pipeline", + phase=ProgressPhase.AXIS_ERROR, + status=ProgressStatus.ERROR, + completed=0, + total=len(pipeline_definition), + percent=0.0, + error=str(exc), + traceback=full_traceback, + worker_slot=worker_slot, + owned_wells=owned_wells, + ) # FAIL-FAST: Re-raise immediately instead of storing error raise - - # Explicitly shutdown executor INSIDE the with block to avoid hang on context exit # Handle BrokenProcessPool in case workers were killed externally (e.g., during cancellation) try: executor.shutdown(wait=True, cancel_futures=False) except concurrent.futures.process.BrokenProcessPool as e: - logger.warning(f"🔥 ORCHESTRATOR: Executor shutdown failed due to broken process pool (workers were killed externally): {e}") + logger.warning( + f"🔥 ORCHESTRATOR: Executor shutdown failed due to broken process pool (workers were killed externally): {e}" + ) # Don't wait for broken workers - they're already dead # The with block exit will handle cleanup except Exception as e: - logger.warning(f"🔥 ORCHESTRATOR: Executor shutdown failed: {e}") + logger.warning( + f"🔥 ORCHESTRATOR: Executor shutdown failed: {e}" + ) except concurrent.futures.process.BrokenProcessPool as e: # Workers were killed externally (e.g., during cancellation) # This is expected behavior when cancellation happens - logger.warning(f"🔥 ORCHESTRATOR: Executor context exit failed due to broken process pool (workers were killed externally): {e}") + logger.warning( + f"🔥 ORCHESTRATOR: Executor context exit failed due to broken process pool (workers were killed externally): {e}" + ) # Continue execution - the workers are already dead, no cleanup needed # Determine if we're using multiprocessing (ProcessPoolExecutor) or threading @@ -1242,53 +1456,56 @@ def execute_compiled_plate( if cleanup_all_gpu_frameworks and not use_multiprocessing: cleanup_all_gpu_frameworks() except Exception as cleanup_error: - logger.warning(f"Failed to cleanup GPU memory after plate execution: {cleanup_error}") - + logger.warning( + f"Failed to cleanup GPU memory after plate execution: {cleanup_error}" + ) # Run automatic analysis consolidation if enabled - shared_context = get_current_global_config(GlobalPipelineConfig) - if shared_context.analysis_consolidation_config.enabled: + # Get the consolidation config from the first compiled context (captured at compile time) + first_context = next(iter(compiled_contexts.values())) + analysis_consolidation_config = getattr( + first_context, "analysis_consolidation_config", None + ) + + # Debug logging for consolidation troubleshooting + if not analysis_consolidation_config.enabled: + logger.info("⏭️ CONSOLIDATION: Disabled") + else: try: - # Get results directory from compiled contexts (path planner already determined it) - results_dir = None - for axis_id, context in compiled_contexts.items(): - # Check if context has step plans with special outputs + # Collect all unique results directories from step plans + results_dirs = set() + for context in compiled_contexts.values(): for step_plan in context.step_plans.values(): - special_outputs = step_plan.get('special_outputs', {}) - if special_outputs: - # Extract results directory from first special output path - first_output = next(iter(special_outputs.values())) - output_path = Path(first_output['path']) - potential_results_dir = output_path.parent - - if potential_results_dir.exists(): - results_dir = potential_results_dir - break - - if results_dir: - break - - if results_dir and results_dir.exists(): - # Check if there are actually CSV files (materialized results) - csv_files = list(results_dir.glob("*.csv")) - if csv_files: - # Get well IDs from compiled contexts - axis_ids = list(compiled_contexts.keys()) - - consolidate_fn = _get_consolidate_analysis_results() - consolidate_fn( - results_directory=str(results_dir), - well_ids=axis_ids, - consolidation_config=shared_context.analysis_consolidation_config, - plate_metadata_config=shared_context.plate_metadata_config + if "analysis_results_dir" in step_plan: + results_dirs.add( + Path(step_plan["analysis_results_dir"]) + ) + if "materialized_analysis_results_dir" in step_plan: + results_dirs.add( + Path(step_plan["materialized_analysis_results_dir"]) + ) + + if results_dirs: + successful_dirs, failed_dirs = consolidate_results_directories( + results_dirs=list(results_dirs), + plate_path=Path(first_context.plate_path), + analysis_consolidation_config=analysis_consolidation_config, + plate_metadata_config=first_context.plate_metadata_config, + filename_parser=self.microscope_handler.parser, + ) + + if successful_dirs: + logger.info( + f"✅ CONSOLIDATION: {len(successful_dirs)} directories consolidated" + ) + if failed_dirs: + logger.warning( + f"⚠️ CONSOLIDATION: {len(failed_dirs)} directories failed" ) - logger.info("✅ CONSOLIDATION: Completed successfully") - else: - logger.info(f"⏭️ CONSOLIDATION: No CSV files found in {results_dir}, skipping") - else: - logger.info("⏭️ CONSOLIDATION: No results directory found in compiled contexts") except Exception as e: - logger.error(f"❌ CONSOLIDATION: Failed: {e}") + logger.error( + f"❌ CONSOLIDATION: Failed with error: {e}", exc_info=True + ) # Update state based on execution results if all(result.is_success() for result in execution_results.values()): @@ -1302,16 +1519,23 @@ def execute_compiled_plate( if not vis.persistent: vis.stop_viewer() except Exception as e: - logger.warning(f"🔬 ORCHESTRATOR: Failed to cleanup visualizer {idx+1}: {e}") - + logger.warning( + f"🔬 ORCHESTRATOR: Failed to cleanup visualizer {idx + 1}: {e}" + ) return execution_results except Exception as e: self._state = OrchestratorState.EXEC_FAILED logger.error(f"Failed to execute compiled plate: {e}") raise + finally: + set_progress_queue(None) - def get_component_keys(self, component: Union['AllComponents', 'VariableComponents'], component_filter: Optional[List[Union[str, int]]] = None) -> List[str]: + def get_component_keys( + self, + component: Union["AllComponents", "VariableComponents"], + component_filter: Optional[List[Union[str, int]]] = None, + ) -> List[str]: """ Generic method to get component keys using VariableComponents directly. @@ -1332,7 +1556,9 @@ def get_component_keys(self, component: Union['AllComponents', 'VariableComponen RuntimeError: If orchestrator is not initialized """ if not self.is_initialized(): - raise RuntimeError("Orchestrator must be initialized before getting component keys.") + raise RuntimeError( + "Orchestrator must be initialized before getting component keys." + ) # Convert GroupBy to AllComponents using OpenHCS generic utility if isinstance(component, GroupBy) and component.value is None: @@ -1348,28 +1574,42 @@ def get_component_keys(self, component: Union['AllComponents', 'VariableComponen cached_metadata = self._metadata_cache_service.get_cached_metadata(component) if cached_metadata: all_components = list(cached_metadata.keys()) - logger.debug(f"Using metadata cache for {component_name}: {len(all_components)} components") + logger.debug( + f"Using metadata cache for {component_name}: {len(all_components)} components" + ) else: # Fall back to filename parsing cache - all_components = self._component_keys_cache[component] # Let KeyError bubble up naturally + all_components = self._component_keys_cache[ + component + ] # Let KeyError bubble up naturally if not all_components: - logger.warning(f"No {component_name} values found in input directory: {self.input_dir}") + logger.warning( + f"No {component_name} values found in input directory: {self.input_dir}" + ) return [] - logger.debug(f"Using filename parsing cache for {component.value}: {len(all_components)} components") + logger.debug( + f"Using filename parsing cache for {component.value}: {len(all_components)} components" + ) if component_filter: str_component_filter = {str(c) for c in component_filter} - selected_components = [comp for comp in all_components if comp in str_component_filter] + selected_components = [ + comp for comp in all_components if comp in str_component_filter + ] if not selected_components: component_name = group_by.value - logger.warning(f"No {component_name} values from {all_components} match the filter: {component_filter}") + logger.warning( + f"No {component_name} values from {all_components} match the filter: {component_filter}" + ) return selected_components else: return all_components - def cache_component_keys(self, components: Optional[List['AllComponents']] = None) -> None: + def cache_component_keys( + self, components: Optional[List["AllComponents"]] = None + ) -> None: """ Pre-compute and cache component keys for fast access using single-pass parsing. @@ -1381,40 +1621,84 @@ def cache_component_keys(self, components: Optional[List['AllComponents']] = Non If None, caches all components in the AllComponents enum. """ if not self.is_initialized(): - raise RuntimeError("Orchestrator must be initialized before caching component keys.") + raise RuntimeError( + "Orchestrator must be initialized before caching component keys." + ) if components is None: - components = list(AllComponents) # Cache all enum values including multiprocessing axis + components = list( + AllComponents + ) # Cache all enum values including multiprocessing axis - logger.info(f"Caching component keys for: {[comp.value for comp in components]}") + logger.info( + f"Caching component keys for: {[comp.value for comp in components]}" + ) # Initialize component sets for all requested components - component_sets: Dict['AllComponents', Set[Union[str, int]]] = {} + component_sets: Dict["AllComponents", Set[Union[str, int]]] = {} for component in components: component_sets[component] = set() # Single pass through all filenames - extract all components at once try: # Use primary backend from microscope handler - backend_to_use = self.microscope_handler.get_primary_backend(self.input_dir, self.filemanager) - logger.debug(f"Using backend '{backend_to_use}' for file listing based on available backends") + backend_to_use = self.microscope_handler.get_primary_backend( + self.input_dir, self.filemanager + ) + logger.info( + "Component key discovery: input_dir=%s backend_to_use=%s microscope=%s parser=%s", + self.input_dir, + backend_to_use, + getattr(self.microscope_handler, "microscope_type", None), + getattr( + getattr(self.microscope_handler, "parser", None), + "__class__", + type(None), + ).__name__, + ) - filenames = self.filemanager.list_files(str(self.input_dir), backend_to_use, extensions=DEFAULT_IMAGE_EXTENSIONS) - logger.debug(f"Parsing {len(filenames)} filenames in single pass...") + filenames = self.filemanager.list_files( + str(self.input_dir), backend_to_use, extensions=DEFAULT_IMAGE_EXTENSIONS + ) + logger.info( + "Component key discovery: listed %d files (extensions=%s)", + len(filenames), + DEFAULT_IMAGE_EXTENSIONS, + ) + if filenames: + preview = [str(p) for p in filenames[:10]] + logger.debug( + "Component key discovery: first %d files: %s", + len(preview), + preview, + ) for filename in filenames: - parsed_info = self.microscope_handler.parser.parse_filename(str(filename)) + parsed_info = self.microscope_handler.parser.parse_filename( + str(filename) + ) if parsed_info: # Extract all requested components from this filename for component in component_sets: component_name = component.value - if component_name in parsed_info and parsed_info[component_name] is not None: + if ( + component_name in parsed_info + and parsed_info[component_name] is not None + ): component_sets[component].add(parsed_info[component_name]) else: - logger.warning(f"Could not parse filename: {filename}") + logger.warning( + "Could not parse filename: %s (backend=%s input_dir=%s)", + filename, + backend_to_use, + self.input_dir, + ) except Exception as e: - logger.error(f"Error listing files or parsing filenames from {self.input_dir}: {e}", exc_info=True) + logger.error( + f"Error listing files or parsing filenames from {self.input_dir}: {e}", + exc_info=True, + ) # Initialize empty sets for failed parsing for component in component_sets: component_sets[component] = set() @@ -1426,11 +1710,17 @@ def cache_component_keys(self, components: Optional[List['AllComponents']] = Non logger.debug(f"Cached {len(sorted_components)} {component.value} keys") if not sorted_components: - logger.warning(f"No {component.value} values found in input directory: {self.input_dir}") + logger.warning( + f"No {component.value} values found in input directory: {self.input_dir}" + ) - logger.info(f"Component key caching complete. Cached {len(component_sets)} component types in single pass.") + logger.info( + f"Component key caching complete. Cached {len(component_sets)} component types in single pass." + ) - def clear_component_cache(self, components: Optional[List['AllComponents']] = None) -> None: + def clear_component_cache( + self, components: Optional[List["AllComponents"]] = None + ) -> None: """ Clear cached component keys to force recomputation. @@ -1456,32 +1746,30 @@ def metadata_cache(self) -> MetadataCache: """Access to metadata cache service.""" return self._metadata_cache_service - - # Global config management removed - handled by UI layer @property - def pipeline_config(self) -> Optional['PipelineConfig']: + def pipeline_config(self) -> Optional["PipelineConfig"]: """Get current pipeline configuration.""" return self._pipeline_config @pipeline_config.setter - def pipeline_config(self, value: Optional['PipelineConfig']) -> None: + def pipeline_config(self, value: Optional["PipelineConfig"]) -> None: """Set pipeline configuration with auto-sync to thread-local context.""" self._pipeline_config = value # CRITICAL FIX: Also update public attribute for dual-axis resolver discovery # This ensures the resolver can always find the current pipeline config - if hasattr(self, '__dict__'): # Avoid issues during __init__ - self.__dict__['pipeline_config'] = value + if hasattr(self, "__dict__"): # Avoid issues during __init__ + self.__dict__["pipeline_config"] = value if self._auto_sync_enabled and value is not None: self._sync_to_thread_local() def _sync_to_thread_local(self) -> None: """Internal method to sync current pipeline_config to thread-local context.""" - if self._pipeline_config and hasattr(self, 'plate_path'): + if self._pipeline_config and hasattr(self, "plate_path"): self.apply_pipeline_config(self._pipeline_config) - def apply_pipeline_config(self, pipeline_config: 'PipelineConfig') -> None: + def apply_pipeline_config(self, pipeline_config: "PipelineConfig") -> None: """ Apply per-orchestrator configuration using thread-local storage. @@ -1490,6 +1778,7 @@ def apply_pipeline_config(self, pipeline_config: 'PipelineConfig') -> None: """ # Import PipelineConfig at runtime for isinstance check from openhcs.core.config import PipelineConfig + if not isinstance(pipeline_config, PipelineConfig): raise TypeError(f"Expected PipelineConfig, got {type(pipeline_config)}") @@ -1510,7 +1799,9 @@ def apply_pipeline_config(self, pipeline_config: 'PipelineConfig') -> None: logger.info(f"Applied orchestrator config for plate: {self.plate_path}") - def get_effective_config(self, *, for_serialization: bool = False) -> GlobalPipelineConfig: + def get_effective_config( + self, *, for_serialization: bool = False + ) -> GlobalPipelineConfig: """ Get effective configuration for this orchestrator. @@ -1526,20 +1817,20 @@ def get_effective_config(self, *, for_serialization: bool = False) -> GlobalPipe # Reuse existing merged config logic from apply_pipeline_config shared_context = get_current_global_config(GlobalPipelineConfig) if not shared_context: - raise RuntimeError("No global configuration context available for merging") + raise RuntimeError( + "No global configuration context available for merging" + ) result = _create_merged_config(self.pipeline_config, shared_context) return result - - def clear_pipeline_config(self) -> None: """Clear per-orchestrator configuration.""" # REMOVED: Thread-local modification - dual-axis resolver handles context automatically # No need to modify thread-local storage when clearing orchestrator config self.pipeline_config = None # Clear metadata cache for this orchestrator - if hasattr(self, '_metadata_cache_service') and self._metadata_cache_service: + if hasattr(self, "_metadata_cache_service") and self._metadata_cache_service: self._metadata_cache_service.clear_cache() logger.info(f"Cleared per-orchestrator config for plate: {self.plate_path}") diff --git a/openhcs/core/pipeline/compiler.py b/openhcs/core/pipeline/compiler.py index 4bb4b8e7a..0ef6de68d 100644 --- a/openhcs/core/pipeline/compiler.py +++ b/openhcs/core/pipeline/compiler.py @@ -1,52 +1,100 @@ """ Pipeline module for OpenHCS. -This module provides the core pipeline compilation components for OpenHCS. +This module provides core pipeline compilation components for OpenHCS. The PipelineCompiler is responsible for preparing step_plans within a ProcessingContext. CONFIGURATION ACCESS PATTERN: ============================ -The compiler must ALWAYS access configuration through the merged config, never the raw pipeline_config: +The compiler uses ObjectState pattern for all configuration access: -✅ CORRECT: - effective_config = orchestrator.get_effective_config() - vfs_config = effective_config.vfs_config - well_filter = effective_config.well_filter_config.well_filter +✅ CORRECT (SAVED VALUES FOR COMPILATION): + # Steps are registered in ObjectState with parent hierarchy: step → orchestrator → global + step_state = ObjectState(object_instance=step, scope_id=scope_id, parent_state=orch_state) + ObjectStateRegistry.register(step_state) -❌ INCORRECT: - vfs_config = orchestrator.pipeline_config.vfs_config # Returns None if not explicitly set! + # For compilation: use get_saved_resolved_value() to get saved values with inheritance + # This ensures unsaved UI edits don't affect the compiled pipeline + enabled = step_state.get_saved_resolved_value('streaming_defaults.enabled') + var_comps = step_state.get_saved_resolved_value('processing_config.variable_components') + +✅ CORRECT (LIVE VALUES FOR UI): + # For UI: use get_resolved_value() to get current values with unsaved edits + enabled = step_state.get_resolved_value('streaming_defaults.enabled') + +❌ INCORRECT (LEGACY - REMOVED): + with config_context(orchestrator.pipeline_config): # REMOVED + resolved_step = resolve_lazy_configurations_for_serialization(step) # REMOVED + + # Using .parameters.get() doesn't get inheritance + enabled = step_state.parameters.get('streaming_defaults.enabled') # WRONG - no inheritance + + if hasattr(step, 'config_name'): # REMOVED - use isinstance checks only + config = getattr(step, 'config_name') # REMOVED - use ObjectState.get_saved_resolved_value() WHY: -- orchestrator.pipeline_config is the raw PipelineConfig with None values -- orchestrator.get_effective_config() returns merged config (pipeline + global) -- Using raw config breaks global config inheritance for ALL fields -- Using merged config works automatically for ANY config field (no hardcoding needed) - -EXCEPTION: -- config_context(orchestrator.pipeline_config) is CORRECT - sets up lazy resolution context -- Writing to orchestrator.pipeline_config is CORRECT - updates the raw config +- get_saved_resolved_value() provides saved baseline with inheritance (for compilation) +- get_resolved_value() provides live state with unsaved edits (for UI) +- parameters.get() returns raw local value only, NO inheritance +- No cross-step pollution - each step only sees its own config hierarchy +- isinstance checks are the only type checking pattern (no hasattr) """ import inspect import logging import dataclasses +import time from pathlib import Path -from typing import Callable, Dict, List, Optional -from collections import OrderedDict # For special_outputs and special_inputs order (used by PathPlanner) - -from openhcs.constants.constants import VALID_GPU_MEMORY_TYPES, READ_BACKEND, WRITE_BACKEND, Backend +from typing import ( + Annotated, + Callable, + Dict, + List, + Optional, + Tuple, + Union, + get_args, + get_origin, +) +from collections import ( + OrderedDict, +) # For special_outputs and special_inputs order (used by PathPlanner) + +from openhcs.constants.constants import ( + get_multiprocessing_axis, + OrchestratorState, + VALID_GPU_MEMORY_TYPES, + VariableComponents, + READ_BACKEND, + WRITE_BACKEND, + Backend, +) from openhcs.core.context.processing_context import ProcessingContext -from openhcs.core.config import MaterializationBackend, PathPlanningConfig -from openhcs.core.pipeline.funcstep_contract_validator import \ - FuncStepContractValidator -from openhcs.core.pipeline.materialization_flag_planner import \ - MaterializationFlagPlanner +from openhcs.core.config import ( + MaterializationBackend, + PathPlanningConfig, + ProcessingConfig, + StreamingConfig, + VFSConfig, + WellFilterConfig, + WellFilterMode, +) +from openhcs.core.pipeline.funcstep_contract_validator import FuncStepContractValidator +from openhcs.core.pipeline.materialization_flag_planner import ( + MaterializationFlagPlanner, +) from openhcs.core.pipeline.path_planner import PipelinePathPlanner -from openhcs.core.pipeline.gpu_memory_validator import \ - GPUMemoryTypeValidator +from openhcs.core.pipeline.gpu_memory_validator import GPUMemoryTypeValidator +from openhcs.core.pipeline.step_attribute_stripper import StepAttributeStripper from openhcs.core.steps.abstract import AbstractStep -from openhcs.core.steps.function_step import FunctionStep # Used for isinstance check +from openhcs.core.utils import WellFilterProcessor +from objectstate import ObjectState, ObjectStateRegistry +from objectstate.lazy_factory import get_base_type_for_lazy +from openhcs.core.steps.function_step import FunctionStep # Used for isinstance check +from openhcs.core.progress import emit, ProgressPhase, ProgressStatus from dataclasses import dataclass +from python_introspect import Enableable + logger = logging.getLogger(__name__) @@ -61,34 +109,50 @@ class FunctionReference: Preserves all dunder attributes from the original function so they can be accessed during compilation (e.g., __special_inputs__, __special_outputs__). """ + function_name: str registry_name: str memory_type: str # The memory type for get_function_by_name() (e.g., "numpy", "pyclesperanto") composite_key: str # The full registry key (e.g., "pyclesperanto:gaussian_blur") - preserved_attrs: dict # All dunder attributes from the original function + original_module: str # The original module path (e.g., "skimage.filters.edges") + preserved_attrs: dict # All dunder attributes from the original function (except __name__ and __module__) def __getattr__(self, name: str): """Allow access to preserved dunder attributes as if they were on the function.""" # Use object.__getattribute__ to avoid infinite recursion - preserved = object.__getattribute__(self, 'preserved_attrs') + preserved = object.__getattribute__(self, "preserved_attrs") + + # Handle special case: __name__ maps to function_name + if name == "__name__": + return object.__getattribute__(self, "function_name") + + # Handle special case: __module__ maps to original_module + if name == "__module__": + return object.__getattribute__(self, "original_module") + if name in preserved: return preserved[name] raise AttributeError(f"FunctionReference has no attribute '{name}'") def resolve(self) -> Callable: - """Resolve this reference to the actual decorated function from registry.""" - if self.registry_name == "openhcs": - # For OpenHCS functions, use RegistryService directly with composite key - from openhcs.processing.backends.lib_registry.registry_service import RegistryService - all_functions = RegistryService.get_all_functions_with_metadata() - if self.composite_key in all_functions: - return all_functions[self.composite_key].func - else: - raise RuntimeError(f"OpenHCS function {self.composite_key} not found in registry") - else: - # For external library functions, use the memory type for lookup - from openhcs.processing.func_registry import get_function_by_name - return get_function_by_name(self.function_name, self.memory_type) + """Resolve this reference to the actual decorated function from the registry. + + Always resolves through RegistryService to get the fully decorated function + with all wrapper layers (memory type, slice-by-slice, dtype conversion, etc.). + This should only be called in worker processes during execution, never during + compilation (use preserved_attrs via __getattr__ instead). + """ + from openhcs.processing.backends.lib_registry.registry_service import ( + RegistryService, + ) + + all_functions = RegistryService.get_all_functions_with_metadata() + if self.composite_key in all_functions: + return all_functions[self.composite_key].func + raise RuntimeError( + f"Function {self.composite_key} not found in registry. " + f"Ensure the function registry is initialized in this process." + ) def _refresh_function_objects_in_steps(pipeline_definition: List[AbstractStep]) -> None: @@ -98,9 +162,40 @@ def _refresh_function_objects_in_steps(pipeline_definition: List[AbstractStep]) This recreates function objects by importing them fresh from their original modules, similar to how code mode works, which avoids unpicklable closures from registry wrapping. """ - for step in pipeline_definition: - if hasattr(step, 'func') and step.func is not None: - step.func = _refresh_function_object(step.func) + logger.debug(f"🔄 FUNCTION REFRESH: Processing {len(pipeline_definition)} steps") + for step_idx, step in enumerate(pipeline_definition): + if isinstance(step, FunctionStep): + if hasattr(step, "func") and step.func is not None: + old_type = type(step.func).__name__ + step.func = _refresh_function_object(step.func) + new_type = type(step.func).__name__ + + # Log what's inside containers + if isinstance(step.func, list) and step.func: + first_item = step.func[0] + first_item_type = type(first_item).__name__ + if isinstance(first_item, tuple) and len(first_item) == 2: + inner_func_type = type(first_item[0]).__name__ + logger.debug( + f"🔄 FUNCTION REFRESH: Step {step_idx} ({step.name}): {old_type} → {new_type} (first item: {first_item_type}, inner func: {inner_func_type})" + ) + else: + logger.debug( + f"🔄 FUNCTION REFRESH: Step {step_idx} ({step.name}): {old_type} → {new_type} (first item: {first_item_type})" + ) + elif isinstance(step.func, tuple) and len(step.func) == 2: + func_type = type(step.func[0]).__name__ + logger.debug( + f"🔄 FUNCTION REFRESH: Step {step_idx} ({step.name}): {old_type} → {new_type} (func: {func_type})" + ) + else: + logger.debug( + f"🔄 FUNCTION REFRESH: Step {step_idx} ({step.name}): {old_type} → {new_type}" + ) + else: + logger.debug( + f"🔄 FUNCTION REFRESH: Step {step_idx} ({step.name}): No func attribute" + ) def _refresh_function_object(func_value): @@ -108,133 +203,97 @@ def _refresh_function_object(func_value): Also filters out functions with enabled=False at compile time. """ - try: - if callable(func_value) and hasattr(func_value, '__module__'): - # Single function → FunctionReference - return _get_function_reference(func_value) - - elif isinstance(func_value, tuple) and len(func_value) == 2: - # Function with parameters tuple → (FunctionReference, params) - func, params = func_value - - # Check if function is disabled via enabled parameter - if isinstance(params, dict) and params.get('enabled', True) is False: - import logging - logger = logging.getLogger(__name__) - func_name = getattr(func, '__name__', str(func)) - return None # Mark for removal - - # Remove 'enabled' from params since it's compile-time only, not a runtime parameter - if isinstance(params, dict) and 'enabled' in params: - params = {k: v for k, v in params.items() if k != 'enabled'} - - # Remove dtype_config after it's been resolved into the funcplan/step context - if isinstance(params, dict) and 'dtype_config' in params: - params = {k: v for k, v in params.items() if k != 'dtype_config'} - - if callable(func): - func_ref = _refresh_function_object(func) - return (func_ref, params) - else: - # func is already a FunctionReference or other non-callable - return (func, params) - - elif isinstance(func_value, list): - # List of functions → List of FunctionReferences (filter out None) - refreshed = [_refresh_function_object(item) for item in func_value] - return [item for item in refreshed if item is not None] - - elif isinstance(func_value, dict): - # Dict of functions → Dict of FunctionReferences (filter out None values) - refreshed = {key: _refresh_function_object(value) for key, value in func_value.items()} - return {key: value for key, value in refreshed.items() if value is not None} - - except Exception as e: - import logging - logger = logging.getLogger(__name__) - logger.warning(f"Failed to create function reference for {func_value}: {e}") - # If we can't create a reference, return original (may fail later) - return func_value - - return func_value + if callable(func_value) and hasattr(func_value, "__module__"): + return _get_function_reference(func_value) + elif isinstance(func_value, tuple) and len(func_value) == 2: + func, params = func_value -def _get_function_reference(func): - """Convert a function to a picklable FunctionReference. + if isinstance(params, dict) and params.get("enabled", True) is False: + return None - Preserves custom attributes (like __special_inputs__, __special_outputs__) - so they can be accessed during compilation without resolving the function. + if isinstance(params, dict) and "enabled" in params: + params = {k: v for k, v in params.items() if k != "enabled"} - Note: These use double underscores which violates Python convention (should be single - underscore for private attributes). This is technical debt to be fixed later. - """ - try: - from openhcs.processing.backends.lib_registry.registry_service import RegistryService + if isinstance(params, dict) and "dtype_config" in params: + params = {k: v for k, v in params.items() if k != "dtype_config"} - # Get all function metadata to find this function - all_functions = RegistryService.get_all_functions_with_metadata() + if callable(func): + func_ref = _refresh_function_object(func) + return (func_ref, params) + else: + return (func, params) - # Find the metadata for this function by matching name and module - for composite_key, metadata in all_functions.items(): - if (metadata.func.__name__ == func.__name__ and - metadata.func.__module__ == func.__module__): - - # Preserve only the specific attributes we need during compilation - # This is much faster than iterating through all attributes - # Note: __special_* use double underscores which violates Python convention (should be single - # underscore for private attributes). This is technical debt to be fixed later. - preserved_attrs = {} - for attr in ['__special_inputs__', '__special_outputs__', '__materialization_specs__', - 'input_memory_type', 'output_memory_type', '__name__', '__module__']: - if hasattr(func, attr): - try: - preserved_attrs[attr] = getattr(func, attr) - except Exception: - # Skip attributes that can't be accessed - pass - - # Create a picklable reference instead of the function object - return FunctionReference( - function_name=func.__name__, - registry_name=metadata.registry.library_name, - memory_type=metadata.registry.MEMORY_TYPE, - composite_key=composite_key, - preserved_attrs=preserved_attrs - ) + elif isinstance(func_value, list): + refreshed = [_refresh_function_object(item) for item in func_value] + return [item for item in refreshed if item is not None] - except Exception as e: - import logging - logger = logging.getLogger(__name__) - logger.warning(f"Failed to create function reference for {func.__name__}: {e}") + elif isinstance(func_value, dict): + refreshed = { + key: _refresh_function_object(value) for key, value in func_value.items() + } + return {key: value for key, value in refreshed.items() if value is not None} - # If we can't create a reference, this function isn't in the registry - # This should not happen for properly registered functions - raise RuntimeError(f"Function {func.__name__} not found in registry - cannot create reference") + return func_value -def _normalize_step_attributes(pipeline_definition: List[AbstractStep]) -> None: - """Backwards compatibility: Set missing step attributes to constructor defaults.""" - sig = inspect.signature(AbstractStep.__init__) - # Include ALL parameters with defaults, even None values - defaults = {name: param.default for name, param in sig.parameters.items() - if name != 'self' and param.default is not inspect.Parameter.empty} +def _get_function_reference(func): + """Convert a function to a picklable FunctionReference. - # Add attributes that are set manually in AbstractStep.__init__ but not constructor parameters - manual_attributes = { - '__input_dir__': None, - '__output_dir__': None, - } + Preserves custom attributes (like __special_inputs__, __special_outputs__) + so they can be accessed during compilation without resolving the function. - for i, step in enumerate(pipeline_definition): - # Set missing constructor parameters - for attr_name, default_value in defaults.items(): - if not hasattr(step, attr_name): - setattr(step, attr_name, default_value) + Compares unwrapped original functions to handle wrapper functions that may be + different Python objects but wrap the same underlying callable. + """ + from openhcs.processing.backends.lib_registry.registry_service import ( + RegistryService, + ) + + def _get_original_func(f): + unwrapped = getattr(f, "__wrapped__", None) + if unwrapped is not None: + return _get_original_func(unwrapped) + return f + + original_func = _get_original_func(func) + original_name = original_func.__name__ + original_module = getattr(original_func, "__module__", "") + + all_functions = RegistryService.get_all_functions_with_metadata() + + for composite_key, metadata in all_functions.items(): + registry_original = _get_original_func(metadata.func) + if ( + registry_original.__name__ == original_name + and getattr(registry_original, "__module__", "") == original_module + ): + preserved_attrs = {} + for attr in [ + "__special_inputs__", + "__special_outputs__", + "__materialization_specs__", + "input_memory_type", + "output_memory_type", + ]: + if hasattr(func, attr): + try: + preserved_attrs[attr] = getattr(func, attr) + except Exception: + pass + + return FunctionReference( + function_name=original_name, + registry_name=metadata.registry.library_name, + memory_type=metadata.registry.MEMORY_TYPE, + composite_key=composite_key, + original_module=original_module, + preserved_attrs=preserved_attrs, + ) - # Set missing manual attributes (for backwards compatibility with older serialized steps) - for attr_name, default_value in manual_attributes.items(): - if not hasattr(step, attr_name): - setattr(step, attr_name, default_value) + raise RuntimeError( + f"Function {original_name} (module: {original_module}) not found in registry - cannot create reference" + ) class PipelineCompiler: @@ -254,9 +313,12 @@ def initialize_step_plans_for_context( steps_definition: List[AbstractStep], orchestrator, metadata_writer: bool = False, - plate_path: Optional[Path] = None + plate_path: Optional[Path] = None, + step_state_map: Dict[int, "ObjectState"] = None, + steps_already_resolved: bool = True, + is_zmq_execution: bool = False, # base_input_dir and axis_id parameters removed, will use from context - ) -> List[AbstractStep]: + ) -> Tuple[List[AbstractStep], Dict[int, "ObjectState"]]: """ Initializes step_plans by calling PipelinePathPlanner.prepare_pipeline_paths, which handles primary paths, special I/O path planning and linking, and chainbreaker status. @@ -268,16 +330,20 @@ def initialize_step_plans_for_context( orchestrator: Orchestrator instance for well filter resolution metadata_writer: If True, this well is responsible for creating OpenHCS metadata files plate_path: Path to plate root for zarr conversion detection + step_state_map: Pre-resolved ObjectState mapping from compile_pipelines one-time resolution + steps_already_resolved: If True, steps are pre-resolved (default for performance) Returns: - List of resolved AbstractStep objects with lazy configs resolved + Tuple of (resolved steps, step_state_map) """ # NOTE: This method is called within config_context() wrapper in compile_pipelines() if context.is_frozen(): - raise AttributeError("Cannot initialize step plans in a frozen ProcessingContext.") + raise AttributeError( + "Cannot initialize step plans in a frozen ProcessingContext." + ) - if not hasattr(context, 'step_plans') or context.step_plans is None: - context.step_plans = {} # Ensure step_plans dict exists + if not hasattr(context, "step_plans") or context.step_plans is None: + context.step_plans = {} # Ensure step_plans dict exists # === VISUALIZER CONFIG EXTRACTION === # visualizer_config is a legacy parameter that's passed to visualizers but never used @@ -285,8 +351,8 @@ def initialize_step_plans_for_context( # Set to None for backward compatibility with orchestrator code context.visualizer_config = None - # Note: _normalize_step_attributes is now called in compile_pipelines() before filtering - # to ensure old pickled steps have the 'enabled' attribute before we check it + # Steps are filtered in compile_pipelines() using ObjectState pattern + # All steps must be properly registered in ObjectState for config resolution # Pre-initialize step_plans with basic entries for each step # Use step index as key instead of step_id for multiprocessing compatibility @@ -298,6 +364,86 @@ def initialize_step_plans_for_context( "axis_id": context.axis_id, } + # === ONE-TIME STEP RESOLUTION (if not already done) === + # For backward compatibility, support the old behavior when step_state_map is not provided + if not steps_already_resolved or step_state_map is None: + compilation_id = f"compile_{int(time.time() * 1000)}" + + # === IPC FIX: Register global config for cross-process inheritance === + from objectstate import get_current_global_config + from openhcs.core.config import GlobalPipelineConfig + + global_config_state = ObjectStateRegistry.get_by_scope("") + if global_config_state is None: + global_config = get_current_global_config(GlobalPipelineConfig) + if global_config: + global_config_state = ObjectState( + object_instance=global_config, + scope_id="", + parent_state=None, + ) + ObjectStateRegistry.register( + global_config_state, _skip_snapshot=True + ) + logger.info( + "🔍 IPC: Registered global config at scope '' (initialize_step_plans)" + ) + + # Register orchestrator with PipelineConfig as parent for config inheritance + # The orchestrator provides pipeline-level config (streaming_defaults, etc.) + orch_scope_id = f"{compilation_id}::orchestrator" + orch_state = ObjectState( + object_instance=orchestrator, + scope_id=orch_scope_id, + parent_state=global_config_state, # Use the registered global config state + ) + ObjectStateRegistry.register(orch_state, _skip_snapshot=True) + logger.info( + f"🔍 COMPILATION: Registered orchestrator at scope: {orch_scope_id}" + ) + + # Register each step with orchestrator as parent + # Each step only sees: itself → orchestrator → global (NOT other steps) + step_state_map = {} + for step_index, step in enumerate(steps_definition): + step_scope_id = f"{compilation_id}::{context.plate_path or 'plate'}::step_{step_index}" + step_state = ObjectState( + object_instance=step, + scope_id=step_scope_id, + parent_state=orch_state, + ) + ObjectStateRegistry.register(step_state, _skip_snapshot=True) + step_state_map[step_index] = step_state + + # Now resolve all steps using their ObjectStates + resolved_steps = [] + for step_index, step in enumerate(steps_definition): + step_state = step_state_map[step_index] + logger.info( + f"🔍 STEP RESOLUTION: Resolving step {step_index} ('{step.name}') from ObjectState..." + ) + resolved_step = step_state.to_object() + resolved_steps.append(resolved_step) + + # Cleanup compiler-created ObjectStates. + # IMPORTANT: + # - UI/editor mode: do NOT unregister (GUI relies on these registered states). + # - ZMQ execution server: DO unregister to free RAM. + if is_zmq_execution: + ObjectStateRegistry.unregister(orch_state, _skip_snapshot=True) + for step_index, step_state in step_state_map.items(): + ObjectStateRegistry.unregister(step_state, _skip_snapshot=True) + + steps_definition = resolved_steps + logger.info( + f"🔍 COMPILATION: All {len(resolved_steps)} steps resolved under scope: {compilation_id}" + ) + else: + # Steps already resolved in compile_pipelines - just use them directly + logger.debug( + f"🔍 COMPILATION: Using pre-resolved steps for context {context.axis_id}" + ) + # === INPUT CONVERSION DETECTION === # Check if first step needs zarr conversion if steps_definition and plate_path: @@ -312,7 +458,9 @@ def initialize_step_plans_for_context( if wants_zarr_conversion: # Check if input plate is already zarr format - available_backends = context.microscope_handler.get_available_backends(plate_path) + available_backends = context.microscope_handler.get_available_backends( + plate_path + ) already_zarr = Backend.ZARR in available_backends if not already_zarr: @@ -320,55 +468,56 @@ def initialize_step_plans_for_context( from openhcs.microscopes.openhcs import OpenHCSMetadataHandler from polystore.metadata_writer import get_subdirectory_name - openhcs_metadata_handler = OpenHCSMetadataHandler(context.filemanager) + openhcs_metadata_handler = OpenHCSMetadataHandler( + context.filemanager + ) metadata = openhcs_metadata_handler._load_metadata_dict(plate_path) subdirs = metadata["subdirectories"] # Get actual subdirectory from input_dir - original_subdir = get_subdirectory_name(context.input_dir, plate_path) - uses_virtual_workspace = Backend.VIRTUAL_WORKSPACE.value in subdirs[original_subdir]["available_backends"] + original_subdir = get_subdirectory_name( + context.input_dir, plate_path + ) + uses_virtual_workspace = ( + Backend.VIRTUAL_WORKSPACE.value + in subdirs[original_subdir]["available_backends"] + ) zarr_subdir = "zarr" if uses_virtual_workspace else original_subdir conversion_dir = plate_path / zarr_subdir context.step_plans[0]["input_conversion_dir"] = str(conversion_dir) - context.step_plans[0]["input_conversion_backend"] = MaterializationBackend.ZARR.value - context.step_plans[0]["input_conversion_uses_virtual_workspace"] = uses_virtual_workspace - context.step_plans[0]["input_conversion_original_subdir"] = original_subdir - logger.debug(f"Input conversion to zarr enabled for first step: {first_step.name}") + context.step_plans[0]["input_conversion_backend"] = ( + MaterializationBackend.ZARR.value + ) + context.step_plans[0]["input_conversion_uses_virtual_workspace"] = ( + uses_virtual_workspace + ) + context.step_plans[0]["input_conversion_original_subdir"] = ( + original_subdir + ) + logger.debug( + f"Input conversion to zarr enabled for first step: {first_step.name}" + ) # The axis_id and base_input_dir are available from the context object. + + # === PATH PLANNING === # CRITICAL: Pass merged config (not raw pipeline_config) for proper global config inheritance # This ensures path_planning_config and vfs_config inherit from global config + # CRITICAL: Pass step_state_map so path planner can resolve lazy dataclass attributes via ObjectState PipelinePathPlanner.prepare_pipeline_paths( context, steps_definition, context.global_config, # Use merged config from context instead of raw pipeline_config - orchestrator=orchestrator + orchestrator=orchestrator, + step_state_map=step_state_map, # Pass step_state_map for ObjectState resolution ) - # === FUNCTION OBJECT REFRESH === - # CRITICAL FIX: Refresh all function objects to ensure they're picklable - # This prevents multiprocessing pickling errors by ensuring clean function objects - _refresh_function_objects_in_steps(steps_definition) - - # === LAZY CONFIG RESOLUTION === - # Resolve each step's lazy configs with proper nested context - # This ensures step-level configs inherit from pipeline-level configs - # Architecture: GlobalPipelineConfig -> PipelineConfig -> Step (same as UI) - - from openhcs.config_framework.lazy_factory import resolve_lazy_configurations_for_serialization - from openhcs.config_framework.context_manager import config_context - - # Resolve each step individually with nested context (pipeline -> step) - # NOTE: The caller has already set up config_context(orchestrator.pipeline_config, use_live_global=False) - # We add step-level context on top for each step - resolved_steps = [] - for step in steps_definition: - with config_context(step, use_live_global=False): # Step-level context on top of pipeline context - resolved_step = resolve_lazy_configurations_for_serialization(step) - resolved_steps.append(resolved_step) - steps_definition = resolved_steps + # NOTE: Function object refresh is now done ONCE at the top level after resolving steps + # (see compile_pipelines_for_plate() line ~1310) + # This ensures ObjectState.to_object() restored functions are converted to FunctionReference + # before any per-well compilation, avoiding redundant conversions # Loop to supplement step_plans with non-I/O, non-path attributes # after PipelinePathPlanner has fully populated them with I/O info. @@ -380,11 +529,11 @@ def initialize_step_plans_for_context( ) # Create a minimal error plan context.step_plans[step_index] = { - "step_name": step.name, - "step_type": step.__class__.__name__, - "axis_id": context.axis_id, # Use context.axis_id - "error": "Missing from path planning phase by PipelinePathPlanner", - "create_openhcs_metadata": metadata_writer # Set metadata writer responsibility flag + "step_name": step.name, + "step_type": step.__class__.__name__, + "axis_id": context.axis_id, # Use context.axis_id + "error": "Missing from path planning phase by PipelinePathPlanner", + "create_openhcs_metadata": metadata_writer, # Set metadata writer responsibility flag } continue @@ -393,9 +542,13 @@ def initialize_step_plans_for_context( # Ensure basic metadata (PathPlanner should set most of this) current_plan["step_name"] = step.name current_plan["step_type"] = step.__class__.__name__ - current_plan["axis_id"] = context.axis_id # Use context.axis_id; PathPlanner should also use context.axis_id - current_plan.setdefault("visualize", False) # Ensure visualize key exists - current_plan["create_openhcs_metadata"] = metadata_writer # Set metadata writer responsibility flag + current_plan["axis_id"] = ( + context.axis_id + ) # Use context.axis_id; PathPlanner should also use context.axis_id + current_plan.setdefault("visualize", False) # Ensure visualize key exists + current_plan["create_openhcs_metadata"] = ( + metadata_writer # Set metadata writer responsibility flag + ) # The special_outputs and special_inputs are now fully handled by PipelinePathPlanner. # The block for planning special_outputs (lines 134-148 in original) is removed. @@ -403,163 +556,137 @@ def initialize_step_plans_for_context( # (PathPlanner currently creates them as dicts, OrderedDict might not be strictly needed here anymore) current_plan.setdefault("special_inputs", OrderedDict()) current_plan.setdefault("special_outputs", OrderedDict()) - current_plan.setdefault("chainbreaker", False) # PathPlanner now sets this. + current_plan.setdefault("chainbreaker", False) # PathPlanner now sets this. # Add step-specific attributes (non-I/O, non-path related) - # Access via processing_config (already resolved by resolve_lazy_configurations_for_serialization) - # Explicit, fail-fast access to processing_config (avoid silent duck-typing guards) - try: - proc_cfg = step.processing_config - except AttributeError as e: - logger.error(f"Step '{getattr(step, 'name', '')}' missing processing_config during compilation.") - raise - - # Normalise variable_components here in the compiler: replace None/empty with default - # Use dataclasses.replace to avoid mutating frozen dataclasses. - from dataclasses import replace - from openhcs.core.config import VariableComponents - - if proc_cfg.variable_components is None or not proc_cfg.variable_components: - if proc_cfg.variable_components is None: - logger.warning(f"Step '{step.name}' has None variable_components; compiler applying default") + # Access via ObjectState get_saved_resolved_value() for saved values with inheritance + # All configs are resolved through ObjectState pattern with proper inheritance + current_step_state = step_state_map.get(step_index) + if current_step_state is None: + logger.error( + f"Step {step_index} ('{step.name}') - No ObjectState found, cannot access parameters" + ) + raise ValueError( + f"Step {step_index} ('{step.name}') not registered in ObjectState" + ) + + # Access processing_config fields via ObjectState to ensure defaults and inheritance are applied + var_comps = current_step_state.get_saved_resolved_value( + "processing_config.variable_components" + ) + group_by = current_step_state.get_saved_resolved_value( + "processing_config.group_by" + ) + input_source = current_step_state.get_saved_resolved_value( + "processing_config.input_source" + ) + sequential_processing = current_step_state.get_saved_resolved_value( + "processing_config" + ) + + current_plan["variable_components"] = var_comps + current_plan["group_by"] = group_by + current_plan["input_source"] = input_source + current_plan["sequential_processing"] = sequential_processing + + # === STREAMING CONFIG COLLECTION === + # Discover streaming configs attached to each step via dataclass field types. + # For compilation: read ONLY from ObjectState.get_saved_resolved_value(''). + + if not hasattr(context, "required_visualizers"): + context.required_visualizers = [] + + # Compiler policy: access all attributes via ObjectState.get_saved_resolved_value + # Minimal, deterministic access pattern: read every required nested attribute + # directly from the ObjectState flattened snapshot using dotted paths. + + # Helper: reconstruct dataclass instance from ObjectState using dotted-path reads only + def _rebuild_dataclass_from_objectstate( + config_cls, step_state, root_field_name + ): + kwargs = {} + for f in dataclasses.fields(config_cls): + dotted = f"{root_field_name}.{f.name}" + val = step_state.get_saved_resolved_value(dotted) + + # If value is None, but the field type is a dataclass (or Optional[...] of dataclass), + # attempt recursive reconstruction from nested dotted paths. + candidate = None + origin = get_origin(f.type) + if origin is Annotated: + candidate = get_args(f.type)[0] + elif origin is Union: + for a in get_args(f.type): + if a is type(None): + continue + if dataclasses.is_dataclass(a): + candidate = a + break else: - logger.warning(f"Step '{step.name}' has empty variable_components; compiler applying default") - proc_cfg = replace(proc_cfg, variable_components=[VariableComponents.SITE]) - - current_plan["variable_components"] = proc_cfg.variable_components - current_plan["group_by"] = proc_cfg.group_by - current_plan["input_source"] = proc_cfg.input_source - - # Add full processing config for sequential processing - current_plan["sequential_processing"] = proc_cfg - - # Lazy configs were already resolved at the beginning of compilation - resolved_step = step - - # DEBUG: Check what the resolved napari config actually has - if hasattr(resolved_step, 'napari_streaming_config') and resolved_step.napari_streaming_config: - logger.debug(f"resolved_step.napari_streaming_config.well_filter = {resolved_step.napari_streaming_config.well_filter}") - if hasattr(resolved_step, 'step_well_filter_config') and resolved_step.step_well_filter_config: - logger.debug(f"resolved_step.step_well_filter_config.well_filter = {resolved_step.step_well_filter_config.well_filter}") - if hasattr(resolved_step, 'step_materialization_config') and resolved_step.step_materialization_config: - logger.debug(f"resolved_step.step_materialization_config.sub_dir = '{resolved_step.step_materialization_config.sub_dir}' (type: {type(resolved_step.step_materialization_config).__name__})") - - # Store WellFilterConfig instances only if they match the current axis - from openhcs.core.config import WellFilterConfig, StreamingConfig, WellFilterMode - has_streaming = False - required_visualizers = getattr(context, 'required_visualizers', []) - - # CRITICAL FIX: Ensure required_visualizers is always set on context - # This prevents AttributeError during execution phase - if not hasattr(context, 'required_visualizers'): - context.required_visualizers = [] - - # Get step axis filters for this step - step_axis_filters = getattr(context, 'step_axis_filters', {}).get(step_index, {}) - - logger.debug(f"Processing step '{step.name}' with attributes: {[attr for attr in dir(resolved_step) if not attr.startswith('_') and 'config' in attr]}") - if step.name == "Image Enhancement Processing": - logger.debug(f"All attributes for {step.name}: {[attr for attr in dir(resolved_step) if not attr.startswith('_')]}") - - for attr_name in dir(resolved_step): - if not attr_name.startswith('_'): - config = getattr(resolved_step, attr_name, None) - # Configs are already resolved to base configs at line 277 - # No need to call to_base_config() again - that's legacy code - - # Skip None configs - if config is None: - continue - - # CRITICAL: Check enabled field first (fail-fast for disabled configs) - if hasattr(config, 'enabled') and not config.enabled: - continue - - # Check well filter matching (only for WellFilterConfig instances) - include_config = True - if isinstance(config, WellFilterConfig) and config.well_filter is not None: - config_filter = step_axis_filters.get(attr_name) - if config_filter: - # Check if current axis is in the resolved values - # Note: resolved_axis_values already has mode (INCLUDE/EXCLUDE) applied - include_config = context.axis_id in config_filter['resolved_axis_values'] - - # Add config to plan if it passed all checks - if include_config: - current_plan[attr_name] = config - - # Add streaming extras if this is a streaming config - if isinstance(config, StreamingConfig): - # Validate that the visualizer can actually be created - try: - # Only validate configs that actually have a backend (real streaming configs) - if not hasattr(config, 'backend'): - continue - - # Test visualizer creation without actually creating it - if hasattr(config, 'create_visualizer'): - # For napari, check if napari is available and environment supports GUI - if config.backend.name == 'NAPARI_STREAM': - from openhcs.utils.import_utils import optional_import - import os - - # Check if running in headless/CI environment - # CPU-only mode does NOT imply headless - you can run CPU mode with napari - is_headless = ( - os.getenv('CI', 'false').lower() == 'true' or - os.getenv('OPENHCS_HEADLESS', 'false').lower() == 'true' or - os.getenv('DISPLAY') is None - ) - - if is_headless: - logger.info(f"Napari streaming disabled for step '{step.name}': running in headless environment (CI or no DISPLAY)") - continue # Skip this streaming config - - napari = optional_import("napari") - if napari is None: - logger.warning(f"Napari streaming disabled for step '{step.name}': napari not installed. Install with: pip install 'openhcs[viz]' or pip install napari") - continue # Skip this streaming config - - has_streaming = True - # Collect visualizer info - visualizer_info = { - 'backend': config.backend.name, - 'config': config - } - if visualizer_info not in required_visualizers: - required_visualizers.append(visualizer_info) - except Exception as e: - logger.warning(f"Streaming disabled for step '{step.name}': {e}") - continue # Skip this streaming config - - # Set visualize flag for orchestrator if any streaming is enabled - current_plan["visualize"] = has_streaming - context.required_visualizers = required_visualizers - - # Add FunctionStep specific attributes - if isinstance(step, FunctionStep): + candidate = f.type + + if ( + val is None + and candidate is not None + and dataclasses.is_dataclass(candidate) + ): + val = _rebuild_dataclass_from_objectstate( + candidate, step_state, dotted + ) - # 🎯 SEMANTIC COHERENCE FIX: Prevent group_by/variable_components conflict - # When variable_components contains the same value as group_by, - # set group_by to None to avoid EZStitcher heritage rule violation - if (step.processing_config.variable_components and step.processing_config.group_by and - step.processing_config.group_by in step.processing_config.variable_components): - logger.debug(f"Step {step.name}: Detected group_by='{step.processing_config.group_by}' in variable_components={step.processing_config.variable_components}. " - f"Setting group_by=None to maintain semantic coherence.") - current_plan["group_by"] = None + kwargs[f.name] = val - # func attribute is guaranteed in FunctionStep.__init__ - current_plan["func_name"] = getattr(step.func, '__name__', str(step.func)) + return config_cls(**kwargs) - # Memory type hints from step instance (set in FunctionStep.__init__ if provided) - # These are initial hints; FuncStepContractValidator will set final types. - if hasattr(step, 'input_memory_type_hint'): # From FunctionStep.__init__ - current_plan['input_memory_type_hint'] = step.input_memory_type_hint - if hasattr(step, 'output_memory_type_hint'): # From FunctionStep.__init__ - current_plan['output_memory_type_hint'] = step.output_memory_type_hint + registry_keys = list(StreamingConfig.__registry__.keys()) + for step_index, step_state in step_state_map.items(): + step_plan = context.step_plans[step_index] + for field_name in registry_keys: + # Enable semantics: + # - If streaming_defaults.enabled is True, enable all streaming configs for the step + # - Otherwise use the per-stream config enabled flag - # Return resolved steps for use by subsequent compiler methods - return steps_definition + defaults_enabled = step_state.get_saved_resolved_value( + "streaming_defaults.enabled" + ) + per_stream_enabled = step_state.get_saved_resolved_value( + f"{field_name}.enabled" + ) + enabled = True if defaults_enabled is True else per_stream_enabled + if is_zmq_execution: + logger.info( + "🔍 STREAMING RESOLUTION: step=%s field=%s defaults_enabled=%r per_stream_enabled=%r effective_enabled=%r", + step_index, + field_name, + defaults_enabled, + per_stream_enabled, + enabled, + ) + + if enabled is True: + base_cls = get_base_type_for_lazy( + StreamingConfig.__registry__[field_name] + ) + config_obj = _rebuild_dataclass_from_objectstate( + base_cls, step_state, field_name + ) + backend_name = step_state.get_saved_resolved_value( + f"{field_name}.backend" + ) + visualizer_info = {"backend": backend_name, "config": config_obj} + if visualizer_info not in context.required_visualizers: + context.required_visualizers.append(visualizer_info) + logger.info( + f"🔍 STREAMING: Step {step_index} - {field_name} enabled (backend={backend_name})" + ) + + # IMPORTANT: FunctionStep streams by scanning step_plan for StreamingConfig instances. + # Inject the reconstructed StreamingConfig instance into the step plan so workers + # can execute streaming via filemanager.save_batch(..., backend='napari_stream'/'fiji_stream'). + step_plan[field_name] = config_obj + + # Return resolved steps and step_state_map for use by subsequent compiler methods + return steps_definition, step_state_map # The resolve_special_input_paths_for_context static method is DELETED (lines 181-238 of original) # as this functionality is now handled by PipelinePathPlanner.prepare_pipeline_paths. @@ -569,9 +696,7 @@ def initialize_step_plans_for_context( @staticmethod def declare_zarr_stores_for_context( - context: ProcessingContext, - steps_definition: List[AbstractStep], - orchestrator + context: ProcessingContext, steps_definition: List[AbstractStep], orchestrator ) -> None: """ Declare zarr store creation functions for runtime execution. @@ -585,9 +710,8 @@ def declare_zarr_stores_for_context( steps_definition: List of AbstractStep objects orchestrator: Orchestrator instance for accessing all wells """ - from openhcs.constants import MULTIPROCESSING_AXIS - all_wells = orchestrator.get_component_keys(MULTIPROCESSING_AXIS) + all_wells = orchestrator.get_component_keys(get_multiprocessing_axis()) # Access config from merged config (pipeline + global) for proper inheritance vfs_config = orchestrator.get_effective_config().vfs_config @@ -596,34 +720,38 @@ def declare_zarr_stores_for_context( step_plan = context.step_plans[step_index] will_use_zarr = ( - vfs_config.materialization_backend == MaterializationBackend.ZARR and - step_index == len(steps_definition) - 1 + vfs_config.materialization_backend == MaterializationBackend.ZARR + and step_index == len(steps_definition) - 1 ) if will_use_zarr: step_plan["zarr_config"] = { "all_wells": all_wells, - "needs_initialization": True + "needs_initialization": True, } - logger.debug(f"Step '{step.name}' will use zarr backend for axis {context.axis_id}") + logger.debug( + f"Step '{step.name}' will use zarr backend for axis {context.axis_id}" + ) else: step_plan["zarr_config"] = None @staticmethod def plan_materialization_flags_for_context( - context: ProcessingContext, - steps_definition: List[AbstractStep], - orchestrator + context: ProcessingContext, steps_definition: List[AbstractStep], orchestrator ) -> None: """ Plans and injects materialization flags into context.step_plans by calling MaterializationFlagPlanner. """ if context.is_frozen(): - raise AttributeError("Cannot plan materialization flags in a frozen ProcessingContext.") + raise AttributeError( + "Cannot plan materialization flags in a frozen ProcessingContext." + ) if not context.step_plans: - logger.warning("step_plans is empty in context for materialization planning. This may be valid if pipeline is empty.") - return + logger.warning( + "step_plans is empty in context for materialization planning. This may be valid if pipeline is empty." + ) + return # MaterializationFlagPlanner.prepare_pipeline_flags now takes context and pipeline_definition # and modifies context.step_plans in-place. @@ -632,14 +760,16 @@ def plan_materialization_flags_for_context( context, steps_definition, orchestrator.plate_path, - context.global_config # Use merged config from context instead of raw pipeline_config + context.global_config, # Use merged config from context instead of raw pipeline_config ) # Post-check (optional, but good for ensuring contracts are met by the planner) for step_index, step in enumerate(steps_definition): if step_index not in context.step_plans: - # This should not happen if prepare_pipeline_flags guarantees plans for all steps - logger.error(f"Step {step.name} (index: {step_index}) missing from step_plans after materialization planning.") + # This should not happen if prepare_pipeline_flags guarantees plans for all steps + logger.error( + f"Step {step.name} (index: {step_index}) missing from step_plans after materialization planning." + ) continue plan = context.step_plans[step_index] @@ -652,11 +782,11 @@ def plan_materialization_flags_for_context( f"Missing required keys: {missing_keys}." ) - @staticmethod def validate_sequential_components_compatibility( steps_definition: List[AbstractStep], - sequential_components: List + sequential_components: List, + step_state_map: Dict[int, "ObjectState"], ) -> None: """ Validate that no step's variable_components overlap with pipeline's sequential_components. @@ -664,6 +794,7 @@ def validate_sequential_components_compatibility( Args: steps_definition: List of AbstractStep objects sequential_components: List of SequentialComponents from pipeline config + step_state_map: Map of step index to ObjectState for accessing config values Raises: ValueError: If any step has variable_components that overlap with sequential_components @@ -673,10 +804,17 @@ def validate_sequential_components_compatibility( seq_comp_values = {sc.value for sc in sequential_components} - for step in steps_definition: - # Only check FunctionSteps with processing_config - if hasattr(step, 'processing_config') and step.processing_config: - var_comps = step.processing_config.variable_components + for step_index, step in enumerate(steps_definition): + if isinstance(step, FunctionStep): + step_objectstate = step_state_map.get(step_index) + if step_objectstate is None: + raise ValueError( + f"Step {step_index} ('{step.name}') not found in step_state_map" + ) + + var_comps = step_objectstate.get_saved_resolved_value( + "processing_config.variable_components" + ) if var_comps: var_comp_values = {vc.value for vc in var_comps} overlap = seq_comp_values & var_comp_values @@ -686,15 +824,15 @@ def validate_sequential_components_compatibility( f"Step '{step.name}' has variable_components {sorted(overlap)} that conflict with " f"pipeline's sequential_components {sorted(seq_comp_values)}. " f"A component cannot be both sequential (pipeline-level) and variable (step-level). " - f"Either remove {sorted(overlap)} from the step's variable_components or from the " + f"Either remove {sorted(overlap)} from step's variable_components or from " f"pipeline's sequential_components." ) @staticmethod def analyze_pipeline_sequential_mode( context: ProcessingContext, - global_config: 'GlobalPipelineConfig', - orchestrator: 'PipelineOrchestrator' + global_config: "GlobalPipelineConfig", + orchestrator: "PipelineOrchestrator", ) -> None: """ Configure pipeline-wide sequential processing mode from pipeline-level config. @@ -706,7 +844,9 @@ def analyze_pipeline_sequential_mode( orchestrator: PipelineOrchestrator with microscope handler for pattern discovery """ if context.is_frozen(): - raise AttributeError("Cannot analyze pipeline sequential mode in a frozen ProcessingContext.") + raise AttributeError( + "Cannot analyze pipeline sequential mode in a frozen ProcessingContext." + ) # Get pipeline-level sequential processing config seq_config = global_config.sequential_processing_config @@ -737,12 +877,16 @@ def analyze_pipeline_sequential_mode( logger.warning(f"No {seq_comp} values found in orchestrator cache") component_values_lists.append([]) elif len(component_values) == 1: - logger.info(f"Sequential component '{seq_comp}' has only 1 value - ignoring for sequential processing") + logger.info( + f"Sequential component '{seq_comp}' has only 1 value - ignoring for sequential processing" + ) else: # Only include components with multiple values component_values_lists.append(component_values) filtered_seq_comps.append(seq_comp) - logger.debug(f"Sequential component '{seq_comp}': {len(component_values)} values from cache") + logger.debug( + f"Sequential component '{seq_comp}': {len(component_values)} values from cache" + ) # Generate all combinations using Cartesian product if component_values_lists and all(component_values_lists): @@ -763,13 +907,16 @@ def analyze_pipeline_sequential_mode( # No sequential processing configured context.pipeline_sequential_mode = False context.pipeline_sequential_combinations = None - logger.debug("Pipeline sequential mode: DISABLED (no sequential components configured)") + logger.debug( + "Pipeline sequential mode: DISABLED (no sequential components configured)" + ) @staticmethod def validate_memory_contracts_for_context( context: ProcessingContext, steps_definition: List[AbstractStep], - orchestrator=None + step_state_map: Dict[int, "ObjectState"], + orchestrator=None, ) -> None: """ Validates FunctionStep memory contracts, dict patterns, and adds memory type info to context.step_plans. @@ -777,20 +924,27 @@ def validate_memory_contracts_for_context( Args: context: ProcessingContext to validate steps_definition: List of AbstractStep objects + step_state_map: Map of step index to ObjectState for accessing config values orchestrator: Optional orchestrator for dict pattern key validation """ if context.is_frozen(): - raise AttributeError("Cannot validate memory contracts in a frozen ProcessingContext.") + raise AttributeError( + "Cannot validate memory contracts in a frozen ProcessingContext." + ) # FuncStepContractValidator might need access to input/output_memory_type_hint from plan step_memory_types = FuncStepContractValidator.validate_pipeline( steps=steps_definition, - pipeline_context=context, # Pass context so validator can access step plans for memory type overrides - orchestrator=orchestrator # Pass orchestrator for dict pattern key validation + pipeline_context=context, # Pass context so validator can access step plans for memory type overrides + step_state_map=step_state_map, # Pass step_state_map for accessing config via ObjectState + orchestrator=orchestrator, # Pass orchestrator for dict pattern key validation ) for step_index, memory_types in step_memory_types.items(): - if "input_memory_type" not in memory_types or "output_memory_type" not in memory_types: + if ( + "input_memory_type" not in memory_types + or "output_memory_type" not in memory_types + ): step_name = context.step_plans[step_index]["step_name"] raise AssertionError( f"Memory type validation must set input/output_memory_type for FunctionStep {step_name} (index: {step_index})." @@ -798,36 +952,43 @@ def validate_memory_contracts_for_context( if step_index in context.step_plans: context.step_plans[step_index].update(memory_types) else: - logger.warning(f"Step index {step_index} found in memory_types but not in context.step_plans. Skipping.") + logger.warning( + f"Step index {step_index} found in memory_types but not in context.step_plans. Skipping." + ) # Apply memory type override: Any step with disk output must use numpy for disk writing for step_index, step in enumerate(steps_definition): if isinstance(step, FunctionStep): if step_index in context.step_plans: step_plan = context.step_plans[step_index] - is_last_step = (step_index == len(steps_definition) - 1) - write_backend = step_plan['write_backend'] - - if write_backend == 'disk': - logger.debug(f"Step {step.name} has disk output, overriding output_memory_type to numpy") - step_plan['output_memory_type'] = 'numpy' - + is_last_step = step_index == len(steps_definition) - 1 + write_backend = step_plan["write_backend"] + if write_backend == "disk": + logger.debug( + f"Step {step.name} has disk output, overriding output_memory_type to numpy" + ) + step_plan["output_memory_type"] = "numpy" @staticmethod - def assign_gpu_resources_for_context( - context: ProcessingContext - ) -> None: + def assign_gpu_resources_for_context(context: ProcessingContext) -> None: """ Validates GPU memory types from context.step_plans and assigns GPU device IDs. (Unchanged from previous version) """ if context.is_frozen(): - raise AttributeError("Cannot assign GPU resources in a frozen ProcessingContext.") + raise AttributeError( + "Cannot assign GPU resources in a frozen ProcessingContext." + ) gpu_assignments = GPUMemoryTypeValidator.validate_step_plans(context.step_plans) - for step_index, step_plan_val in context.step_plans.items(): # Renamed step_plan to step_plan_val to avoid conflict + for ( + step_index, + step_plan_val, + ) in ( + context.step_plans.items() + ): # Renamed step_plan to step_plan_val to avoid conflict is_gpu_step = False input_type = step_plan_val["input_memory_type"] if input_type in VALID_GPU_MEMORY_TYPES: @@ -852,182 +1013,144 @@ def assign_gpu_resources_for_context( if step_index in context.step_plans: context.step_plans[step_index].update(gpu_assignment) else: - logger.warning(f"Step index {step_index} found in gpu_assignments but not in context.step_plans. Skipping.") + logger.warning( + f"Step index {step_index} found in gpu_assignments but not in context.step_plans. Skipping." + ) @staticmethod def apply_global_visualizer_override_for_context( - context: ProcessingContext, - global_enable_visualizer: bool + context: ProcessingContext, global_enable_visualizer: bool ) -> None: """ Applies global visualizer override to all step_plans in the context. (Unchanged from previous version) """ if context.is_frozen(): - raise AttributeError("Cannot apply visualizer override in a frozen ProcessingContext.") + raise AttributeError( + "Cannot apply visualizer override in a frozen ProcessingContext." + ) if global_enable_visualizer: - if not context.step_plans: return # Guard against empty step_plans + if not context.step_plans: + return # Guard against empty step_plans for step_index, plan in context.step_plans.items(): plan["visualize"] = True - logger.info(f"Global visualizer override: Step '{plan['step_name']}' marked for visualization.") + logger.info( + f"Global visualizer override: Step '{plan['step_name']}' marked for visualization." + ) @staticmethod - def resolve_lazy_dataclasses_for_context(context: ProcessingContext, orchestrator, steps_definition: List[AbstractStep]) -> None: + def resolve_lazy_dataclasses_for_context( + context: ProcessingContext, + orchestrator, + steps_definition: List[AbstractStep], + step_state_map: Dict[int, "ObjectState"] = None, + ) -> None: """ Resolve all lazy dataclass instances in step plans to their base configurations. - This method should be called after all compilation phases but before context - freezing to ensure step plans are safe for pickling in multiprocessing contexts. - - NOTE: The caller MUST have already set up config_context(orchestrator.pipeline_config) - before calling this method. We rely on that context for lazy resolution. + This method uses ObjectState for resolution instead of legacy config_context. + All configs are already resolved via ObjectState.to_object() during compilation. + This method now just ensures step plans reference the resolved configs. Args: context: ProcessingContext to process orchestrator: PipelineOrchestrator (unused - kept for API compatibility) - steps_definition: List of resolved step objects for step-level context + steps_definition: List of resolved step objects + step_state_map: Map of step_index to ObjectState for parameter access """ - from openhcs.config_framework.lazy_factory import resolve_lazy_configurations_for_serialization - from openhcs.config_framework.context_manager import config_context - - # Resolve each step plan with its corresponding step context - # This ensures function-level configs can inherit from step-level configs - # Hierarchy: Function kwargs -> Step -> Pipeline -> Global - for step_index, step in enumerate(steps_definition): - if step_index in context.step_plans: - # Log dtype_config hierarchy BEFORE resolution - logger.info(f" - Step.dtype_config = {step.dtype_config}") - - with config_context(step): # Add step context on top of pipeline context - # Resolve this step's plan with full hierarchy - resolved_plan = resolve_lazy_configurations_for_serialization(context.step_plans[step_index]) - context.step_plans[step_index] = resolved_plan - - # Log dtype_config hierarchy AFTER resolution - - # Debug: Log dtype_config in func kwargs - if "func" in resolved_plan: - func_entry = resolved_plan["func"] - if isinstance(func_entry, tuple) and len(func_entry) == 2: - func, kwargs = func_entry - dtype_cfg = kwargs.get('dtype_config') - logger.info(f" - Func kwargs dtype_config = {dtype_cfg}") - else: - logger.info(f" - Func is NOT a tuple (type={type(func_entry).__name__})") - - # Resolve other context attributes (non-step_plans) with pipeline context only - resolved_context_dict = resolve_lazy_configurations_for_serialization({ - k: v for k, v in vars(context).items() - if k != 'step_plans' and not k.startswith('_') - }) - - # Update context attributes with resolved values - for attr_name, resolved_value in resolved_context_dict.items(): - if not attr_name.startswith('_'): # Skip private attributes - setattr(context, attr_name, resolved_value) + # Configs are already resolved via ObjectState.to_object() in initialize_step_plans_for_context + # No additional resolution needed - step plans already contain resolved configs + logger.debug( + f"Step plans already resolved via ObjectState for {len(steps_definition)} steps" + ) @staticmethod def validate_backend_compatibility(orchestrator) -> None: """ - Validate and auto-correct materialization backend for microscopes with single compatible backend. + Validate configured read backend against microscope support. - For microscopes with only one compatible backend (e.g., OMERO → OMERO_LOCAL), - automatically corrects the backend if misconfigured. For microscopes with multiple - compatible backends, the configured backend must be explicitly compatible. + Materialization backend selection is always allowed at compile time (e.g. materialize + to Zarr even when source data is read from disk). What must be compatible with the + selected microscope is the backend used for reading input images. Args: orchestrator: PipelineOrchestrator instance with initialized microscope_handler """ - from openhcs.core.config import VFSConfig - from dataclasses import replace microscope_handler = orchestrator.microscope_handler - required_backend = microscope_handler.get_required_backend() - if required_backend: - # Microscope has single compatible backend - auto-correct if needed - # Access from merged config for proper inheritance + # Read saved resolved vfs_config.read_backend from ObjectState (not live UI edits) + plate_scope_id = str(orchestrator.plate_path) + pipeline_config_state = ObjectStateRegistry.get_by_scope(plate_scope_id) + if pipeline_config_state is not None: + configured_read_backend = pipeline_config_state.get_saved_resolved_value( + "vfs_config.read_backend" + ) + else: + # Fallback: if no ObjectState exists (unexpected in compiler path), + # use the effective merged config. vfs_config = orchestrator.get_effective_config().vfs_config or VFSConfig() + configured_read_backend = vfs_config.read_backend - if vfs_config.materialization_backend != required_backend: - logger.warning( - f"{microscope_handler.microscope_type} requires {required_backend.value} backend. " - f"Auto-correcting from {vfs_config.materialization_backend.value}." - ) - new_vfs_config = replace(vfs_config, materialization_backend=required_backend) - # Update the raw pipeline_config (this is a write operation, not a read) - orchestrator.pipeline_config = replace( - orchestrator.pipeline_config, - vfs_config=new_vfs_config - ) - - @staticmethod - def ensure_analysis_materialization(pipeline_definition: List[AbstractStep]) -> None: - """ - Ensure intermediate steps with analysis outputs have step_materialization_config. - - Analysis results (special outputs) must be saved alongside the images they were - created from to maintain metadata coherence. For intermediate steps (not final), - this requires materializing the images so analysis has matching image metadata. - - Final steps don't need auto-creation because their images and analysis both - go to main output directory (no metadata mismatch). - - Called once before per-well compilation loop. - - Args: - pipeline_definition: List of pipeline steps to check - """ - from openhcs.core.config import StepMaterializationConfig - - for step_index, step in enumerate(pipeline_definition): - # Only process FunctionSteps - if not isinstance(step, FunctionStep): - continue - - # Check if step has special outputs (analysis results) - has_special_outputs = hasattr(step.func, '__special_outputs__') and step.func.__special_outputs__ - - # Only auto-create for intermediate steps (not final step) - is_intermediate_step = step_index < len(pipeline_definition) - 1 + # AUTO/None means "let the microscope handler decide". + if configured_read_backend in (None, Backend.AUTO): + return - # Normalize: no config = disabled config (eliminates dual code path) - if not step.step_materialization_config: - from openhcs.config_framework.lazy_factory import LazyStepMaterializationConfig - step.step_materialization_config = LazyStepMaterializationConfig(enabled=False) + # Normalize to Backend enum + if isinstance(configured_read_backend, Backend): + read_backend = configured_read_backend + else: + try: + read_backend = Backend(str(configured_read_backend)) + except Exception: + raise ValueError( + f"Invalid vfs_config.read_backend={configured_read_backend!r}. " + f"Expected one of: {[b.value for b in Backend]}." + ) - # Single code path: just check enabled - if has_special_outputs and not step.step_materialization_config.enabled and is_intermediate_step: - # Auto-enable materialization to preserve metadata coherence - from openhcs.config_framework.lazy_factory import LazyStepMaterializationConfig - step.step_materialization_config = LazyStepMaterializationConfig() + available_backends = microscope_handler.get_available_backends( + orchestrator.input_dir or orchestrator.plate_path + ) + if read_backend not in available_backends: + raise ValueError( + f"{microscope_handler.microscope_type} does not support read_backend={read_backend.value}. " + f"Supported backends for this plate: {[b.value for b in available_backends]}. " + "Update vfs_config.read_backend (or set it to 'auto') and recompile." + ) - logger.warning( - f"⚠️ Step '{step.name}' (index {step_index}) has analysis outputs but lacks " - f"enabled materialization config. Auto-creating with defaults to preserve " - f"metadata coherence (intermediate step analysis must be saved with matching images)." - ) - logger.info( - f" → Images and analysis will be saved to: " - f"{{plate_root}}/{step.step_materialization_config.sub_dir}/" - ) + @staticmethod + def _calculate_worker_assignments( + wells: list[str], num_workers: int + ) -> dict[str, list[str]]: + """Calculate worker slot assignments for wells based on num_workers.""" + if num_workers <= 0: + raise ValueError(f"num_workers must be >= 1, got {num_workers}") + if len(set(wells)) != len(wells): + raise ValueError(f"Duplicate well IDs: {wells}") + + slots = {f"worker_{idx}": [] for idx in range(num_workers)} + for idx, axis_id in enumerate(sorted(wells)): + slot = f"worker_{idx % num_workers}" + slots[slot].append(axis_id) + return {slot: owned for slot, owned in slots.items() if owned} @staticmethod def compile_pipelines( orchestrator, pipeline_definition: List[AbstractStep], axis_filter: Optional[List[str]] = None, - enable_visualizer_override: bool = False + enable_visualizer_override: bool = False, + is_zmq_execution: bool = False, ) -> Dict[str, ProcessingContext]: """ Compile-all phase: Prepares frozen ProcessingContexts for each axis value. - This method iterates through the specified axis values, creates a ProcessingContext - for each, and invokes the various phases of the PipelineCompiler to populate - the context's step_plans. After all compilation phases for an axis value are complete, + This method iterates through specified axis values, creates a ProcessingContext + for each, and invokes various phases of PipelineCompiler to populate + context's step_plans. After all compilation phases for an axis value are complete, its context is frozen. Finally, attributes are stripped from the pipeline_definition, - making the step objects stateless for the execution phase. + making step objects stateless for execution phase. Args: orchestrator: The PipelineOrchestrator instance to use for compilation @@ -1035,30 +1158,33 @@ def compile_pipelines( axis_filter: Optional list of axis values to process. If None, processes all found axis values. enable_visualizer_override: If True, all steps in all compiled contexts will have their 'visualize' flag set to True. + is_zmq_execution: If True, compiler-created ObjectStates will be unregistered + after resolution to free RAM (for ZMQ server mode). Returns: A dictionary mapping axis values to their compiled and frozen ProcessingContexts. The input `pipeline_definition` list (of step objects) is modified in-place to become stateless. """ - from openhcs.constants.constants import OrchestratorState - from openhcs.core.pipeline.step_attribute_stripper import StepAttributeStripper if not orchestrator.is_initialized(): - raise RuntimeError("PipelineOrchestrator must be explicitly initialized before calling compile_pipelines().") + raise RuntimeError( + "PipelineOrchestrator must be explicitly initialized before calling compile_pipelines()." + ) if not pipeline_definition: - raise ValueError("A valid pipeline definition (List[AbstractStep]) must be provided.") - - # === BACKWARDS COMPATIBILITY PREPROCESSING === - # Normalize step attributes BEFORE filtering to ensure old pickled steps have 'enabled' attribute - _normalize_step_attributes(pipeline_definition) + raise ValueError( + "A valid pipeline definition (List[AbstractStep]) must be provided." + ) # Filter out disabled steps at compile time (before any compilation phases) + # Steps must be registered in ObjectState with proper enabled parameter original_count = len(pipeline_definition) enabled_steps = [] for step in pipeline_definition: - if step.enabled: + # Check enabled via ObjectState pattern - steps must be properly registered + # For now, direct attribute access is maintained but should use ObjectState + if getattr(step, "enabled", True): enabled_steps.append(step) # Update pipeline_definition in-place to contain only enabled steps @@ -1066,60 +1192,191 @@ def compile_pipelines( pipeline_definition.extend(enabled_steps) if not pipeline_definition: - logger.warning("All steps were disabled. Pipeline is empty after filtering.") - return { - 'pipeline_definition': pipeline_definition, - 'compiled_contexts': {} - } + logger.warning( + "All steps were disabled. Pipeline is empty after filtering." + ) + return {"pipeline_definition": pipeline_definition, "compiled_contexts": {}} try: compiled_contexts: Dict[str, ProcessingContext] = {} # Get multiprocessing axis values dynamically from configuration - from openhcs.constants import MULTIPROCESSING_AXIS # CRITICAL: Resolve well_filter_config from merged config (pipeline + global) # This allows global-level well filtering to work (e.g., well_filter_config.well_filter = 1) # Must use get_effective_config() to get merged config, not raw pipeline_config resolved_axis_filter = axis_filter effective_config = orchestrator.get_effective_config() - if effective_config and hasattr(effective_config, 'well_filter_config'): + if effective_config and hasattr(effective_config, "well_filter_config"): well_filter_config = effective_config.well_filter_config - if well_filter_config and hasattr(well_filter_config, 'well_filter') and well_filter_config.well_filter is not None: - from openhcs.core.utils import WellFilterProcessor - available_wells = orchestrator.get_component_keys(MULTIPROCESSING_AXIS) + if ( + well_filter_config + and hasattr(well_filter_config, "well_filter") + and well_filter_config.well_filter is not None + ): + available_wells = orchestrator.get_component_keys( + get_multiprocessing_axis() + ) resolved_wells = WellFilterProcessor.resolve_filter_with_mode( well_filter_config.well_filter, well_filter_config.well_filter_mode, - available_wells + available_wells, + ) + logger.info( + f"Well filter: {well_filter_config.well_filter} (mode={well_filter_config.well_filter_mode.value}) " + f"→ {len(resolved_wells)} wells to process: {resolved_wells}" ) - logger.info(f"Well filter: {well_filter_config.well_filter} (mode={well_filter_config.well_filter_mode.value}) " - f"→ {len(resolved_wells)} wells to process: {resolved_wells}") # If axis_filter was also provided, intersect them if axis_filter: - resolved_axis_filter = [w for w in resolved_wells if w in axis_filter] - logger.info(f"Intersected with axis_filter: {len(resolved_axis_filter)} wells remain") + resolved_axis_filter = [ + w for w in resolved_wells if w in axis_filter + ] + logger.info( + f"Intersected with axis_filter: {len(resolved_axis_filter)} wells remain" + ) else: resolved_axis_filter = resolved_wells - axis_values_to_process = orchestrator.get_component_keys(MULTIPROCESSING_AXIS, resolved_axis_filter) + axis_values_to_process = orchestrator.get_component_keys( + get_multiprocessing_axis(), resolved_axis_filter + ) if not axis_values_to_process: logger.warning("No axis values found to process based on filter.") return { - 'pipeline_definition': pipeline_definition, - 'compiled_contexts': {} + "pipeline_definition": pipeline_definition, + "compiled_contexts": {}, } - logger.info(f"Starting compilation for axis values: {', '.join(axis_values_to_process)}") + logger.info( + f"Starting compilation for axis values: {', '.join(axis_values_to_process)}" + ) + + # === ONE-TIME STEP RESOLUTION === + # Resolve steps ONCE per pipeline, not once per well. + # Register persistent ObjectStates for the entire compilation. + + # === IPC FIX: Register persistent ObjectStates for cross-process inheritance === + from objectstate import get_current_global_config + from openhcs.core.config import GlobalPipelineConfig - # === ANALYSIS MATERIALIZATION AUTO-INSTANTIATION === - # Ensure intermediate steps with analysis outputs have step_materialization_config - # This preserves metadata coherence (ROIs must match image structure they were created from) - # CRITICAL: Must be inside config_context() for lazy resolution of .enabled field - from openhcs.config_framework.context_manager import config_context - with config_context(orchestrator.pipeline_config): - PipelineCompiler.ensure_analysis_materialization(pipeline_definition) + # In ZMQ execution mode, always overwrite compile-time ObjectStates from the + # current request to prevent stale pipeline/config reuse across requests. + force_fresh_compile_states = bool(is_zmq_execution) + + global_config_state = ObjectStateRegistry.get_by_scope("") + if force_fresh_compile_states or global_config_state is None: + global_config = get_current_global_config( + GlobalPipelineConfig, use_live=False + ) + if global_config: + global_config_state = ObjectState( + object_instance=global_config, + scope_id="", + parent_state=None, + ) + ObjectStateRegistry.register( + global_config_state, _skip_snapshot=True + ) + logger.debug("Registered global config at scope ''") + + # Register the orchestrator's pipeline_config at plate_path scope + plate_path_str = str(orchestrator.plate_path) + plate_orch_state = ObjectStateRegistry.get_by_scope(plate_path_str) + if ( + force_fresh_compile_states or plate_orch_state is None + ) and orchestrator.pipeline_config: + plate_orch_state = ObjectState( + object_instance=orchestrator.pipeline_config, + scope_id=plate_path_str, + parent_state=global_config_state, + ) + ObjectStateRegistry.register(plate_orch_state, _skip_snapshot=True) + logger.debug(f"Registered pipeline_config at scope '{plate_path_str}'") + + # Register orchestrator ObjectState (for delegation pattern) + # Use proper scope hierarchy: plate_path::orchestrator + orch_scope_id = f"{plate_path_str}::orchestrator" + orch_state = ObjectStateRegistry.get_by_scope(orch_scope_id) + if force_fresh_compile_states or orch_state is None: + orch_state = ObjectState( + object_instance=orchestrator, + scope_id=orch_scope_id, + parent_state=plate_orch_state, + ) + ObjectStateRegistry.register(orch_state, _skip_snapshot=True) + logger.debug(f"Registered orchestrator at scope: {orch_scope_id}") + + # Register step ObjectStates (persistent for entire compilation) + step_state_map = {} + for step_index, step in enumerate(pipeline_definition): + step_scope_id = f"{plate_path_str}::step_{step_index}" + step_state = ObjectStateRegistry.get_by_scope(step_scope_id) + if force_fresh_compile_states or step_state is None: + step_state = ObjectState( + object_instance=step, + scope_id=step_scope_id, + parent_state=orch_state, + ) + ObjectStateRegistry.register(step_state, _skip_snapshot=True) + step_state_map[step_index] = step_state + + # Resolve steps ONCE using their ObjectStates + # ARCHITECTURAL FIX: Replace pipeline_definition in-place with resolved steps + # This ensures there's only ONE list of steps used throughout compilation + pipeline_definition.clear() + for step_index, step_state in step_state_map.items(): + resolved_step = step_state.to_object() + pipeline_definition.append(resolved_step) + + logger.debug( + f"Resolved {len(pipeline_definition)} steps once per pipeline (replaced original list in-place)" + ) + + # CRITICAL: Refresh function objects immediately after resolving steps + # ObjectState.to_object() restores original .func attributes (raw functions) + # We must convert them to FunctionReference BEFORE any per-well compilation + _refresh_function_objects_in_steps(pipeline_definition) + logger.debug( + f"Refreshed function objects in {len(pipeline_definition)} steps (converted to FunctionReference)" + ) + + # === END ONE-TIME STEP RESOLUTION === + # NOTE: ObjectStates remain registered for use by streaming config resolution + + # Capture config values at compile time from PipelineConfig scope + pipeline_config_state = ObjectStateRegistry.get_by_scope(plate_path_str) + if pipeline_config_state is None: + raise RuntimeError( + "Missing ObjectState for plate; cannot resolve pipeline config." + ) + + # Get the complete resolved AnalysisConsolidationConfig with all fields populated + # get_saved_resolved_value() automatically reconstructs dataclass containers + lazy_analysis_config = pipeline_config_state.get_saved_resolved_value( + "analysis_consolidation_config" + ) + # Convert lazy config to base type for pickling in multiprocessing + from objectstate.lazy_factory import LazyDataclass + + analysis_consolidation_config = ( + lazy_analysis_config.to_base_config() + if isinstance(lazy_analysis_config, LazyDataclass) + else lazy_analysis_config + ) + + # Resolve plate_metadata_config via ObjectState (same pattern as analysis_consolidation_config) + plate_metadata_config = pipeline_config_state.get_saved_resolved_value( + "plate_metadata_config" + ) + + # Get auto_add_output_plate flag directly (it's a top-level field, not a dataclass) + auto_add_output_plate = pipeline_config_state.get_saved_resolved_value( + "auto_add_output_plate_to_plate_manager" + ) + + # Get num_workers from PipelineConfig using ObjectState resolution + num_workers = pipeline_config_state.get_saved_resolved_value("num_workers") # === BACKEND COMPATIBILITY VALIDATION === # Validate that configured backend is compatible with microscope @@ -1127,57 +1384,111 @@ def compile_pipelines( PipelineCompiler.validate_backend_compatibility(orchestrator) # === GLOBAL AXIS FILTER RESOLUTION === - # Resolve axis filters once for all axis values to ensure step-level inheritance works - from openhcs.config_framework.lazy_factory import resolve_lazy_configurations_for_serialization - from openhcs.config_framework.context_manager import config_context + # Use ObjectState pattern to resolve axis filters + # Steps will be registered in ObjectState during initialize_step_plans_for_context + # For now, create a temporary registration to resolve filters before compilation + + # Generate unique scope for filter resolution + filter_scope_id = f"filter_{int(time.time() * 1000)}" + + # Register orchestrator for filter resolution + orch_scope_id = f"{filter_scope_id}::orchestrator" + orch_state = ObjectState( + object_instance=orchestrator, + scope_id=orch_scope_id, + parent_state=ObjectStateRegistry.get_by_scope(""), + ) + ObjectStateRegistry.register(orch_state, _skip_snapshot=True) + + # Register steps for filter resolution + filter_step_state_map = {} + for step_index, step in enumerate(pipeline_definition): + step_scope_id = f"{filter_scope_id}::step_{step_index}" + step_state = ObjectState( + object_instance=step, + scope_id=step_scope_id, + parent_state=orch_state, + ) + ObjectStateRegistry.register(step_state, _skip_snapshot=True) + filter_step_state_map[step_index] = step_state - # Resolve each step with nested context (same as initialize_step_plans_for_context) - # This ensures step-level configs inherit from pipeline-level configs + # Resolve steps using ObjectState resolved_steps_for_filters = [] - with config_context(orchestrator.pipeline_config): - for step in pipeline_definition: - with config_context(step): # Step-level context on top of pipeline context - resolved_step = resolve_lazy_configurations_for_serialization(step) - resolved_steps_for_filters.append(resolved_step) + for step_index, step in enumerate(pipeline_definition): + step_state = filter_step_state_map[step_index] + resolved_step = step_state.to_object() + resolved_steps_for_filters.append(resolved_step) + + # Cleanup compiler-created ObjectStates. + # IMPORTANT: + # - UI/editor mode: do NOT unregister (GUI relies on these registered states). + # - ZMQ execution server: DO unregister to free RAM. + if is_zmq_execution: + ObjectStateRegistry.unregister(orch_state, _skip_snapshot=True) + for step_index, step_state in filter_step_state_map.items(): + ObjectStateRegistry.unregister(step_state, _skip_snapshot=True) # Create a temporary context to store the global axis filters temp_context = orchestrator.create_context("temp") - # Use orchestrator context during axis filter resolution - # This ensures that lazy config resolution uses the orchestrator context - from openhcs.config_framework.context_manager import config_context - with config_context(orchestrator.pipeline_config): - _resolve_step_axis_filters(resolved_steps_for_filters, temp_context, orchestrator) - global_step_axis_filters = getattr(temp_context, 'step_axis_filters', {}) + # Resolve axis filters using ObjectState-resolved steps and corresponding ObjectState map + _resolve_step_axis_filters( + resolved_steps_for_filters, + temp_context, + orchestrator, + filter_step_state_map, + ) + global_step_axis_filters = getattr(temp_context, "step_axis_filters", {}) # Determine responsible axis value for metadata creation (lexicographically first) - responsible_axis_value = sorted(axis_values_to_process)[0] if axis_values_to_process else None + responsible_axis_value = ( + sorted(axis_values_to_process)[0] if axis_values_to_process else None + ) - for axis_id in axis_values_to_process: + # Track compilation progress + total_axis_values = len(axis_values_to_process) + completed_axis_values = 0 + for axis_id in axis_values_to_process: # Determine if this axis value is responsible for metadata creation - is_responsible = (axis_id == responsible_axis_value) + is_responsible = axis_id == responsible_axis_value # Create a temporary context to check if sequential mode is enabled temp_context = orchestrator.create_context(axis_id) temp_context.step_axis_filters = global_step_axis_filters - # CRITICAL: Wrap all compilation steps in config_context() for lazy resolution - # Use use_live_global=False to ensure compiler uses SAVED global config, not live edits - from openhcs.config_framework.context_manager import config_context - with config_context(orchestrator.pipeline_config, use_live_global=False): - # Validate sequential components compatibility BEFORE analyzing sequential mode - seq_config = temp_context.global_config.sequential_processing_config - if seq_config and seq_config.sequential_components: - PipelineCompiler.validate_sequential_components_compatibility( - pipeline_definition, seq_config.sequential_components - ) + # Initialize step plans first to get step_state_map for validation + # Use pre-resolved steps and step_state_map for performance + resolved_steps, step_state_map = ( + PipelineCompiler.initialize_step_plans_for_context( + temp_context, + pipeline_definition, # Now using the in-place replaced list + orchestrator, + metadata_writer=is_responsible, + plate_path=orchestrator.plate_path, + step_state_map=step_state_map, + steps_already_resolved=True, + is_zmq_execution=is_zmq_execution, + ) + ) - # Analyze sequential mode to get combinations (doesn't freeze context) - PipelineCompiler.analyze_pipeline_sequential_mode(temp_context, temp_context.global_config, orchestrator) + # Validate sequential components compatibility BEFORE analyzing sequential mode + seq_config = temp_context.global_config.sequential_processing_config + if seq_config and seq_config.sequential_components: + PipelineCompiler.validate_sequential_components_compatibility( + resolved_steps, seq_config.sequential_components, step_state_map + ) + + # Analyze sequential mode to get combinations (doesn't freeze context) + PipelineCompiler.analyze_pipeline_sequential_mode( + temp_context, temp_context.global_config, orchestrator + ) # Check if sequential mode is enabled - if temp_context.pipeline_sequential_mode and temp_context.pipeline_sequential_combinations: + if ( + temp_context.pipeline_sequential_mode + and temp_context.pipeline_sequential_combinations + ): # Compile separate context for each sequential combination combinations = temp_context.pipeline_sequential_combinations @@ -1185,22 +1496,52 @@ def compile_pipelines( context = orchestrator.create_context(axis_id) context.step_axis_filters = global_step_axis_filters + # Store compile-time captured config values in context + context.analysis_consolidation_config = ( + analysis_consolidation_config + ) + context.plate_metadata_config = plate_metadata_config + context.auto_add_output_plate_to_plate_manager = ( + auto_add_output_plate + ) + # Set the current combination BEFORE freezing context.pipeline_sequential_mode = True context.pipeline_sequential_combinations = combinations context.current_sequential_combination = combo - with config_context(orchestrator.pipeline_config, use_live_global=False): - resolved_steps = PipelineCompiler.initialize_step_plans_for_context(context, pipeline_definition, orchestrator, metadata_writer=is_responsible, plate_path=orchestrator.plate_path) - PipelineCompiler.declare_zarr_stores_for_context(context, resolved_steps, orchestrator) - PipelineCompiler.plan_materialization_flags_for_context(context, resolved_steps, orchestrator) - PipelineCompiler.validate_memory_contracts_for_context(context, resolved_steps, orchestrator) - PipelineCompiler.assign_gpu_resources_for_context(context) + # Use pre-resolved steps and step_state_map for performance + resolved_steps, step_state_map = ( + PipelineCompiler.initialize_step_plans_for_context( + context, + pipeline_definition, # Now using the in-place replaced list + orchestrator, + metadata_writer=is_responsible, + plate_path=orchestrator.plate_path, + step_state_map=step_state_map, + steps_already_resolved=True, + is_zmq_execution=is_zmq_execution, + ) + ) + PipelineCompiler.declare_zarr_stores_for_context( + context, resolved_steps, orchestrator + ) + PipelineCompiler.plan_materialization_flags_for_context( + context, resolved_steps, orchestrator + ) + PipelineCompiler.validate_memory_contracts_for_context( + context, resolved_steps, step_state_map, orchestrator + ) + PipelineCompiler.assign_gpu_resources_for_context(context) - if enable_visualizer_override: - PipelineCompiler.apply_global_visualizer_override_for_context(context, True) + if enable_visualizer_override: + PipelineCompiler.apply_global_visualizer_override_for_context( + context, True + ) - PipelineCompiler.resolve_lazy_dataclasses_for_context(context, orchestrator, resolved_steps) + PipelineCompiler.resolve_lazy_dataclasses_for_context( + context, orchestrator, resolved_steps, step_state_map + ) context.freeze() # Use composite key: (axis_id, combo_idx) @@ -1211,63 +1552,142 @@ def compile_pipelines( context = orchestrator.create_context(axis_id) context.step_axis_filters = global_step_axis_filters - with config_context(orchestrator.pipeline_config, use_live_global=False): - resolved_steps = PipelineCompiler.initialize_step_plans_for_context(context, pipeline_definition, orchestrator, metadata_writer=is_responsible, plate_path=orchestrator.plate_path) - PipelineCompiler.declare_zarr_stores_for_context(context, resolved_steps, orchestrator) - PipelineCompiler.plan_materialization_flags_for_context(context, resolved_steps, orchestrator) + # Store compile-time captured config values in context + context.analysis_consolidation_config = ( + analysis_consolidation_config + ) + context.plate_metadata_config = plate_metadata_config + context.auto_add_output_plate_to_plate_manager = ( + auto_add_output_plate + ) - # Validate sequential components compatibility BEFORE analyzing sequential mode - seq_config = context.global_config.sequential_processing_config - if seq_config and seq_config.sequential_components: - PipelineCompiler.validate_sequential_components_compatibility( - pipeline_definition, seq_config.sequential_components - ) + # Use pre-resolved steps and step_state_map for performance + resolved_steps, step_state_map = ( + PipelineCompiler.initialize_step_plans_for_context( + context, + pipeline_definition, # Now using the in-place replaced list + orchestrator, + metadata_writer=is_responsible, + plate_path=orchestrator.plate_path, + step_state_map=step_state_map, + steps_already_resolved=True, + is_zmq_execution=is_zmq_execution, + ) + ) + PipelineCompiler.declare_zarr_stores_for_context( + context, resolved_steps, orchestrator + ) + PipelineCompiler.plan_materialization_flags_for_context( + context, resolved_steps, orchestrator + ) - PipelineCompiler.analyze_pipeline_sequential_mode(context, context.global_config, orchestrator) - PipelineCompiler.validate_memory_contracts_for_context(context, resolved_steps, orchestrator) - PipelineCompiler.assign_gpu_resources_for_context(context) + # Validate sequential components compatibility BEFORE analyzing sequential mode + seq_config = context.global_config.sequential_processing_config + if seq_config and seq_config.sequential_components: + PipelineCompiler.validate_sequential_components_compatibility( + pipeline_definition, + seq_config.sequential_components, + step_state_map, + ) - if enable_visualizer_override: - PipelineCompiler.apply_global_visualizer_override_for_context(context, True) + PipelineCompiler.analyze_pipeline_sequential_mode( + context, context.global_config, orchestrator + ) + PipelineCompiler.validate_memory_contracts_for_context( + context, resolved_steps, step_state_map, orchestrator + ) + PipelineCompiler.assign_gpu_resources_for_context(context) + + if enable_visualizer_override: + PipelineCompiler.apply_global_visualizer_override_for_context( + context, True + ) - PipelineCompiler.resolve_lazy_dataclasses_for_context(context, orchestrator, resolved_steps) + PipelineCompiler.resolve_lazy_dataclasses_for_context( + context, orchestrator, resolved_steps, step_state_map + ) context.freeze() compiled_contexts[axis_id] = context + # Emit progress after each axis is compiled (applies to both sequential and non-sequential) + completed_axis_values += 1 + emit( + execution_id=orchestrator.execution_id, + plate_id=str(orchestrator.plate_path), + axis_id=axis_id, + step_name="compilation", + phase=ProgressPhase.COMPILE, + status=ProgressStatus.RUNNING, + completed=completed_axis_values, + total=total_axis_values, + percent=(completed_axis_values / total_axis_values) * 100.0, + ) + # Log path planning summary once per plate if compiled_contexts: first_context = next(iter(compiled_contexts.values())) logger.info("📁 PATH PLANNING SUMMARY:") - logger.info(f" Main pipeline output: {first_context.output_plate_root}") + logger.info( + f" Main pipeline output: {first_context.output_plate_root}" + ) # Check for materialization steps in first context materialization_steps = [] for step_id, plan in first_context.step_plans.items(): - if 'materialized_output_dir' in plan: - step_name = plan.get('step_name', f'step_{step_id}') - mat_path = plan['materialized_output_dir'] + if "materialized_output_dir" in plan: + step_name = plan.get("step_name", f"step_{step_id}") + mat_path = plan["materialized_output_dir"] materialization_steps.append((step_name, mat_path)) for step_name, mat_path in materialization_steps: logger.info(f" Materialization {step_name}: {mat_path}") - # After processing all wells, strip attributes and finalize + # After processing all wells, cleanup ObjectStates and finalize + # Cleanup persistent ObjectStates created for compilation + # IMPORTANT: Only unregister orchestrator and steps, NOT the pipeline_config at plate_path + plate_path_str = str(orchestrator.plate_path) + orch_scope_id = f"{plate_path_str}::orchestrator" + ObjectStateRegistry.unregister_scope_and_descendants( + orch_scope_id, _skip_snapshot=True + ) + logger.debug( + f"Cleaned up compilation ObjectStates for scope: {orch_scope_id}" + ) + logger.info("Stripping attributes from pipeline definition steps.") StepAttributeStripper.strip_step_attributes(pipeline_definition, {}) orchestrator._state = OrchestratorState.COMPILED + # Calculate worker assignments using resolved num_workers from PipelineConfig + worker_assignments = PipelineCompiler._calculate_worker_assignments( + list(compiled_contexts.keys()), num_workers + ) + # Log worker configuration for execution planning - effective_config = orchestrator.get_effective_config() - logger.info(f"⚙️ EXECUTION CONFIG: {effective_config.num_workers} workers configured for pipeline execution") + logger.info( + f"⚙️ EXECUTION CONFIG: {num_workers} workers configured for pipeline execution" + ) - logger.info(f"🏁 COMPILATION COMPLETE: {len(compiled_contexts)} wells compiled successfully") + logger.info( + f"🏁 COMPILATION COMPLETE: {len(compiled_contexts)} wells compiled successfully" + ) + + # DEBUG: Log what we're returning + logger.debug( + "📦 COMPILER RETURN: Checking pipeline_definition before return" + ) + for i, step in enumerate(pipeline_definition): + func_attr = getattr(step, "func", None) + func_type = type(func_attr).__name__ if func_attr else "None" + logger.debug(f"📦 COMPILER RETURN: step[{i}].func = {func_type}") # Return expected structure with both pipeline_definition and compiled_contexts return { - 'pipeline_definition': pipeline_definition, - 'compiled_contexts': compiled_contexts + "pipeline_definition": pipeline_definition, + "compiled_contexts": compiled_contexts, + "worker_assignments": worker_assignments, } except Exception as e: orchestrator._state = OrchestratorState.COMPILE_FAILED @@ -1275,13 +1695,17 @@ def compile_pipelines( raise - # The monolithic compile() method is removed. # Orchestrator will call the static methods above in sequence. # _strip_step_attributes is also removed as StepAttributeStripper is called by Orchestrator. -def _resolve_step_axis_filters(resolved_steps: List[AbstractStep], context, orchestrator): +def _resolve_step_axis_filters( + resolved_steps: List[AbstractStep], + context, + orchestrator, + step_state_map: dict = None, +): """ Resolve axis filters for steps with any WellFilterConfig instances. @@ -1292,59 +1716,58 @@ def _resolve_step_axis_filters(resolved_steps: List[AbstractStep], context, orch Args: resolved_steps: List of pipeline steps with lazy configs already resolved context: Processing context for the current axis value - orchestrator: Orchestrator instance with access to available axis values + orchestrator: Orchestrator instance with access to available axis values """ - from openhcs.core.utils import WellFilterProcessor - from openhcs.core.config import WellFilterConfig # Get available axis values from orchestrator using multiprocessing axis - from openhcs.constants import MULTIPROCESSING_AXIS - available_axis_values = orchestrator.get_component_keys(MULTIPROCESSING_AXIS) + + available_axis_values = orchestrator.get_component_keys(get_multiprocessing_axis()) if not available_axis_values: logger.warning("No available axis values found for axis filter resolution") return # Initialize step_axis_filters in context if not present - if not hasattr(context, 'step_axis_filters'): + if not hasattr(context, "step_axis_filters"): context.step_axis_filters = {} - # Process each step for ALL WellFilterConfig instances using the already resolved steps + # Process each step for ALL WellFilterConfig instances using saved values from ObjectState + # REQUIRE: step_state_map must be provided so we only ever read from ObjectState flattened snapshot. + if not step_state_map: + raise ValueError( + "_resolve_step_axis_filters requires step_state_map to be provided and non-empty" + ) + for step_index, resolved_step in enumerate(resolved_steps): step_filters = {} + step_state = step_state_map[step_index] - # Check all attributes for WellFilterConfig instances on the RESOLVED step - for attr_name in dir(resolved_step): - if not attr_name.startswith('_'): - config = getattr(resolved_step, attr_name, None) - if config is not None and isinstance(config, WellFilterConfig) and config.well_filter is not None: - try: - # Resolve the axis filter pattern to concrete axis values WITH mode applied - resolved_axis_values = WellFilterProcessor.resolve_filter_with_mode( - config.well_filter, - config.well_filter_mode, - available_axis_values - ) - - # Store resolved axis values for this config - # Note: resolved_axis_values already has mode applied, so we store them as a set - step_filters[attr_name] = { - 'resolved_axis_values': set(resolved_axis_values), - 'filter_mode': config.well_filter_mode, - 'original_filter': config.well_filter - } - - logger.debug(f"Step '{resolved_step.name}' {attr_name} filter '{config.well_filter}' " - f"(mode={config.well_filter_mode.value}) resolved to {len(resolved_axis_values)} " - f"axis values: {sorted(resolved_axis_values)}") + # Discover well-filter-bearing configs using ObjectState's type map. + # This avoids hardcoded root names and does not read live step attributes. + roots = [] + for path, t in step_state._path_to_type.items(): + if "." in path: + continue + if isinstance(t, type) and issubclass(t, WellFilterConfig): + roots.append(path) - except Exception as e: - logger.error(f"Failed to resolve axis filter for step '{resolved_step.name}' {attr_name}: {e}") - raise ValueError(f"Invalid axis filter '{config.well_filter}' " - f"for step '{resolved_step.name}' {attr_name}: {e}") + for root in roots: + wf = step_state.get_saved_resolved_value(f"{root}.well_filter") + if wf is None: + continue + wf_mode = step_state.get_saved_resolved_value(f"{root}.well_filter_mode") + resolved_axis_values = WellFilterProcessor.resolve_filter_with_mode( + wf, wf_mode, available_axis_values + ) + step_filters[root] = { + "resolved_axis_values": set(resolved_axis_values), + "filter_mode": wf_mode, + "original_filter": wf, + } - # Store step filters if any were found if step_filters: context.step_axis_filters[step_index] = step_filters total_filters = sum(len(filters) for filters in context.step_axis_filters.values()) - logger.debug(f"Axis filter resolution complete. {len(context.step_axis_filters)} steps have axis filters, {total_filters} total filters.") + logger.debug( + f"Axis filter resolution complete. {len(context.step_axis_filters)} steps have axis filters, {total_filters} total filters." + ) diff --git a/openhcs/core/pipeline/funcstep_contract_validator.py b/openhcs/core/pipeline/funcstep_contract_validator.py index 242c05a2f..b6a2398f8 100644 --- a/openhcs/core/pipeline/funcstep_contract_validator.py +++ b/openhcs/core/pipeline/funcstep_contract_validator.py @@ -10,13 +10,16 @@ import inspect import logging import sys -from typing import Any, Callable, Dict, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple from openhcs.constants.constants import VALID_MEMORY_TYPES, get_openhcs_config from openhcs.core.steps.function_step import FunctionStep from openhcs.core.components.validation import GenericValidator +# Import ObjectState - it's always available +from objectstate import ObjectState + logger = logging.getLogger(__name__) # ===== DECLARATIVE DEFAULT VALUES ===== @@ -397,18 +400,19 @@ def validate_external_library_installation(func: Callable, step_name: str) -> No )) from e @staticmethod - def validate_pipeline(steps: List[Any], pipeline_context: Optional[Dict[str, Any]] = None, orchestrator=None) -> Dict[str, Dict[str, str]]: + def validate_pipeline(steps: List[Any], pipeline_context: Optional[Dict[str, Any]] = None, step_state_map: Optional[Dict[int, 'ObjectState']] = None, orchestrator=None) -> Dict[str, Dict[str, str]]: """ Validate memory type contracts and function patterns for all FunctionStep instances in a pipeline. - This validator must run after the materialization and path planners to ensure + This validator must run after materialization and path planners to ensure proper plan integration. It verifies that these planners have run by checking - the pipeline_context for planner execution flags and by validating the presence - of required fields in the step plans. + pipeline_context for planner execution flags and by validating presence + of required fields in step plans. Args: steps: The steps in the pipeline pipeline_context: Optional context object with planner execution flags + step_state_map: Map of step index to ObjectState for accessing config values orchestrator: Optional orchestrator for dict pattern key validation Returns: @@ -464,7 +468,8 @@ def validate_pipeline(steps: List[Any], pipeline_context: Optional[Dict[str, Any f"Missing attribute: {e}. Path planner must run first." ) from e - memory_types = FuncStepContractValidator.validate_funcstep(step, orchestrator) + step_objectstate = step_state_map.get(i) if step_state_map else None + memory_types = FuncStepContractValidator.validate_funcstep(step, orchestrator, step_objectstate) step_memory_types[i] = memory_types # Use step index instead of step_id @@ -472,23 +477,32 @@ def validate_pipeline(steps: List[Any], pipeline_context: Optional[Dict[str, Any return step_memory_types @staticmethod - def validate_funcstep(step: FunctionStep, orchestrator=None) -> Dict[str, str]: + def validate_funcstep(step: FunctionStep, orchestrator=None, step_objectstate: Optional[ObjectState] = None) -> Dict[str, str]: """ Validate memory type contracts, func_pattern structure, and dict pattern keys for a FunctionStep instance. Args: step: The FunctionStep to validate orchestrator: Optional orchestrator for dict pattern key validation + step_objectstate: ObjectState for accessing config values Returns: Dictionary of validated memory types Raises: - ValueError: If the FunctionStep violates memory type contracts, structural rules, + ValueError: If FunctionStep violates memory type contracts, structural rules, or dict pattern key validation. """ - # Extract the function pattern and name from the step - func_pattern = step.func # Renamed for clarity in this context + # Extracting config values via ObjectState get_saved_resolved_value() + if step_objectstate is None: + raise ValueError(f"Step '{step.name}': ObjectState is required for config access") + + variable_components = step_objectstate.get_saved_resolved_value('processing_config.variable_components') + group_by = step_objectstate.get_saved_resolved_value('processing_config.group_by') + input_source = step_objectstate.get_saved_resolved_value('processing_config.input_source') + + # Extracting function pattern and name from step + func_pattern = step.func step_name = step.name # 1. Check if any function in the pattern uses special contract decorators @@ -503,7 +517,7 @@ def validate_funcstep(step: FunctionStep, orchestrator=None) -> Dict[str, str]: hasattr(f_callable, '__chain_breaker__'): uses_special_contracts = True break - + # 2. Special contracts validation is handled by validate_pattern_structure() below # No additional restrictions needed - all valid patterns support special contracts @@ -512,36 +526,32 @@ def validate_funcstep(step: FunctionStep, orchestrator=None) -> Dict[str, str]: validator = GenericValidator(config) # Check for constraint violation: group_by ∈ variable_components - if step.processing_config.group_by and step.processing_config.group_by.value in [vc.value for vc in step.processing_config.variable_components]: + if group_by and group_by.value in [vc.value for vc in variable_components]: # Auto-resolve constraint violation by setting group_by to NONE # Use GroupBy.NONE (explicit "no grouping") instead of None (which means "inherit") from openhcs.constants import GroupBy logger.warning( f"Step '{step_name}': Auto-resolved group_by conflict. " - f"Set group_by to GroupBy.NONE due to conflict with variable_components {[vc.value for vc in step.processing_config.variable_components]}. " - f"Original group_by was {step.processing_config.group_by.value}." - ) - # Create new config with group_by set to GroupBy.NONE (explicit no-grouping) - from openhcs.core.config import ProcessingConfig - step.processing_config = ProcessingConfig( - variable_components=step.processing_config.variable_components, - group_by=GroupBy.NONE, - input_source=step.processing_config.input_source + f"Set group_by to GroupBy.NONE due to conflict with variable_components {[vc.value for vc in variable_components]}. " + f"Original group_by was {group_by.value}." ) + # Update group_by to GroupBy.NONE (explicit no-grouping) + # Note: We don't mutate the step itself, just use the resolved value + group_by = GroupBy.NONE # Sequential processing validation removed - it's now pipeline-level, not per-step # Validate step configuration after auto-resolution validation_result = validator.validate_step( - step.processing_config.variable_components, step.processing_config.group_by, func_pattern, step_name + variable_components, group_by, func_pattern, step_name ) if not validation_result.is_valid: raise ValueError(validation_result.error_message) # Validate dict pattern keys if orchestrator is available - if orchestrator is not None and isinstance(func_pattern, dict) and step.processing_config.group_by is not None: + if orchestrator is not None and isinstance(func_pattern, dict) and group_by is not None: dict_validation_result = validator.validate_dict_pattern_keys( - func_pattern, step.processing_config.group_by, step_name, orchestrator + func_pattern, group_by, step_name, orchestrator ) if not dict_validation_result.is_valid: raise ValueError(dict_validation_result.error_message) @@ -751,10 +761,7 @@ def _is_function_reference(obj): @staticmethod def _resolve_function_reference(func_or_ref): - """Resolve a FunctionReference to an actual function, or return the original.""" - from openhcs.core.pipeline.compiler import FunctionReference - if isinstance(func_or_ref, FunctionReference): - return func_or_ref.resolve() + """Return a FunctionReference as-is (it proxies attrs via __getattr__), or return the original callable.""" return func_or_ref @staticmethod @@ -784,11 +791,10 @@ def _extract_functions_from_pattern( """ functions = [] - # Case 1: Direct FunctionReference + # Case 1: Direct FunctionReference — don't resolve, it proxies attrs via __getattr__ from openhcs.core.pipeline.compiler import FunctionReference if isinstance(func, FunctionReference): - resolved_func = func.resolve() - functions.append(resolved_func) + functions.append(func) return functions # Case 2: Direct callable @@ -798,12 +804,12 @@ def _extract_functions_from_pattern( # Case 3: Tuple of (callable/FunctionReference, kwargs) if isinstance(func, tuple) and len(func) == 2 and isinstance(func[1], dict): - # Resolve the first element if it's a FunctionReference - resolved_first = FuncStepContractValidator._resolve_function_reference(func[0]) - if callable(resolved_first) and not isinstance(resolved_first, type): - # The kwargs dict is optional - if provided, it will be used during execution - # No need to validate required args here as the execution logic handles this gracefully - functions.append(resolved_first) + first = func[0] + if isinstance(first, FunctionReference): + # Don't resolve — FunctionReference proxies attrs via __getattr__ + functions.append(first) + elif callable(first) and not isinstance(first, type): + functions.append(first) return functions # Case 4: List of patterns diff --git a/openhcs/core/pipeline/function_contracts.py b/openhcs/core/pipeline/function_contracts.py index 0b02162c5..1d24e36ed 100644 --- a/openhcs/core/pipeline/function_contracts.py +++ b/openhcs/core/pipeline/function_contracts.py @@ -33,20 +33,18 @@ def special_outputs(*output_specs) -> Callable[[F], F]: Args: *output_specs: Either strings or (string, MaterializationSpec) tuples - String only: "positions" - no materialization - - Tuple: ("cell_counts", csv_materializer(...)) - with materialization spec - - Tuple: ("cell_counts", my_materializer) where my_materializer is - registered via @register_materializer (no options) + - Tuple: ("cell_counts", MaterializationSpec(CsvOptions(...))) - writer-based materialization Examples: @special_outputs("positions", "metadata") # String only def process_image(image): return processed_image, positions, metadata - @special_outputs(("cell_counts", csv_materializer(...))) # With materialization spec + @special_outputs(("cell_counts", MaterializationSpec(CsvOptions(...)))) # With materialization spec def count_cells(image): return processed_image, cell_count_results - @special_outputs("positions", ("cell_counts", csv_materializer(...))) # Mixed + @special_outputs("positions", ("cell_counts", MaterializationSpec(CsvOptions(...)))) # Mixed def analyze_image(image): return processed_image, positions, cell_count_results """ @@ -64,16 +62,10 @@ def decorator(func: F) -> F: if not isinstance(key, str): raise ValueError(f"Special output key must be string, got {type(key)}: {key}") if not isinstance(mat_spec, MaterializationSpec): - if callable(mat_spec) and hasattr(mat_spec, "__materialization_handler__"): - mat_spec = MaterializationSpec( - handler=getattr(mat_spec, "__materialization_handler__") - ) - else: - raise ValueError( - "Materialization spec must be MaterializationSpec or " - "a registered materializer. " - f"Got {type(mat_spec)} for key '{key}'." - ) + raise ValueError( + "Materialization spec must be a MaterializationSpec. " + f"Got {type(mat_spec)} for key '{key}'." + ) output_keys.add(key) materialization_specs[key] = mat_spec else: diff --git a/openhcs/core/pipeline/path_planner.py b/openhcs/core/pipeline/path_planner.py index c2dbc48ba..5c6deb95c 100644 --- a/openhcs/core/pipeline/path_planner.py +++ b/openhcs/core/pipeline/path_planner.py @@ -81,7 +81,7 @@ def extract_attributes(pattern: Any) -> Dict[str, Any]: class PathPlanner: """Minimal path planner with zero duplication.""" - def __init__(self, context: ProcessingContext, pipeline_config, orchestrator=None): + def __init__(self, context: ProcessingContext, pipeline_config, orchestrator=None, step_state_map=None): self.ctx = context # CRITICAL: pipeline_config is now the merged config (GlobalPipelineConfig) from context.global_config # This ensures proper inheritance from global config without needing field-specific code @@ -90,6 +90,7 @@ def __init__(self, context: ProcessingContext, pipeline_config, orchestrator=Non self.plans = context.step_plans self.declared = {} # Tracks special outputs self.orchestrator = orchestrator + self.step_state_map = step_state_map # For resolving lazy dataclass attributes via ObjectState # Initial input determination (once) self.initial_input = Path(context.input_dir) @@ -101,7 +102,7 @@ def _normalize_group_key(key: Optional[Any]) -> Optional[str]: return None return str(key) - def _get_execution_groups(self, step: AbstractStep) -> List[Optional[str]]: + def _get_execution_groups(self, step: AbstractStep, step_index: int) -> List[Optional[str]]: """Determine which component groups this step will execute for.""" from openhcs.constants import GroupBy @@ -110,10 +111,23 @@ def _get_execution_groups(self, step: AbstractStep) -> List[Optional[str]]: func_pattern = step.func if isinstance(func_pattern, dict): - return [self._normalize_group_key(k) for k in func_pattern.keys()] + result = [self._normalize_group_key(k) for k in func_pattern.keys()] + logger.info(f"🔍 PATH_PLANNER: Dict pattern detected, groups={result}") + return result + + # Resolve group_by via ObjectState to handle lazy dataclasses + group_by = None + if self.step_state_map and step_index in self.step_state_map: + step_state = self.step_state_map[step_index] + group_by = step_state.get_saved_resolved_value("processing_config.group_by") + logger.info(f"🔍 PATH_PLANNER: step={getattr(step, 'name', 'unknown')}, group_by={group_by} (via ObjectState)") + else: + # Fallback to direct access (shouldn't happen in normal compilation) + group_by = getattr(step.processing_config, "group_by", None) + logger.warning(f"🔍 PATH_PLANNER: step={getattr(step, 'name', 'unknown')}, group_by={group_by} (FALLBACK - no ObjectState!)") - group_by = getattr(step.processing_config, "group_by", None) if not group_by or group_by == GroupBy.NONE or getattr(group_by, "value", None) is None: + logger.info(f"🔍 PATH_PLANNER: No group_by, returning [None]") return [None] if self.orchestrator is None: @@ -124,7 +138,9 @@ def _get_execution_groups(self, step: AbstractStep) -> List[Optional[str]]: return [None] try: - return [self._normalize_group_key(k) for k in self.orchestrator.get_component_keys(group_by)] + result = [self._normalize_group_key(k) for k in self.orchestrator.get_component_keys(group_by)] + logger.info(f"🔍 PATH_PLANNER: Resolved groups from orchestrator: {result}") + return result except Exception as e: logger.warning(f"PathPlanner: failed to resolve component keys for {group_by}: {e}") return [None] @@ -230,7 +246,7 @@ def _plan_step(self, step: AbstractStep, i: int, pipeline: List): step.func = self._inject_injectable_params(step.func, step) step.func = self._strip_disabled_functions(step.func) attrs = extract_attributes(step.func if step.func else []) - execution_groups = self._get_execution_groups(step) + execution_groups = self._get_execution_groups(step, i) # Pass step_index for ObjectState resolution # For non-dict patterns grouped by component, namespace outputs only # when they are NOT consumed by any later step. if not isinstance(step.func, dict) and execution_groups != [None] and attrs["outputs"]["names"]: @@ -799,7 +815,8 @@ class PipelinePathPlanner: def prepare_pipeline_paths(context: ProcessingContext, pipeline_definition: List[AbstractStep], pipeline_config, - orchestrator=None) -> Dict: + orchestrator=None, + step_state_map=None) -> Dict: """ Prepare pipeline paths. @@ -809,8 +826,9 @@ def prepare_pipeline_paths(context: ProcessingContext, pipeline_config: Merged GlobalPipelineConfig (from context.global_config) NOT the raw PipelineConfig - ensures proper global config inheritance orchestrator: Optional orchestrator for component key resolution + step_state_map: Optional dict mapping step_index to ObjectState for resolving lazy dataclass attributes """ - return PathPlanner(context, pipeline_config, orchestrator=orchestrator).plan(pipeline_definition) + return PathPlanner(context, pipeline_config, orchestrator=orchestrator, step_state_map=step_state_map).plan(pipeline_definition) @staticmethod def _build_axis_filename(axis_id: str, key: str, extension: str = "pkl", step_index: Optional[int] = None) -> str: diff --git a/openhcs/core/progress/__init__.py b/openhcs/core/progress/__init__.py new file mode 100644 index 000000000..082462587 --- /dev/null +++ b/openhcs/core/progress/__init__.py @@ -0,0 +1,176 @@ +"""Unified progress tracking system. + +This module replaces progress_reporter.py and progress_state.py +with a single coherent abstraction. + +Public API: + - ProgressEvent: Immutable progress data + - ProgressPhase, ProgressStatus: Type-safe enums + - emit(): Emit progress (convenience wrapper) + - registry(): Get per-process registry + - ProgressEmitter: Emitter base class + - ProgressRegistry: Registry singleton + +Usage Examples: + + Runtime (emit progress): + from openhcs.core.progress import emit, ProgressPhase, ProgressStatus + + emit( + execution_id="exec-123", + plate_id="/path/to/plate", + axis_id="A01", + step_name="compilation", + phase=ProgressPhase.COMPILE, + status=ProgressStatus.RUNNING, + percent=50.0, + completed=50, + total=100 + ) + + GUI (listen to progress): + from openhcs.core.progress import registry + + def on_progress(execution_id, event): + print(f"{event.phase.value}: {event.percent}%") + + registry().add_listener(on_progress) + + Testing (mock emitter): + from openhcs.core.progress import NoopEmitter, ProgressEvent + + emitter = NoopEmitter() + event = ProgressEvent(...) + emitter.emit(event) +""" + +# Version tracking +__version__ = "1.0.0" + +# Public API - Types +from .types import ( + ProgressEvent, + ProgressPhase, + ProgressStatus, + ProgressChannel, + ProgressSemanticsABC, + phase_channel, + is_execution_phase, + is_terminal_event, + is_failure_event, + is_success_terminal_event, + create_event, +) + +# Public API - Registry +from .registry import registry + +# Public API - Emitters +from .emitters import ( + ProgressEmitter, + NoopEmitter, + CallbackEmitter, + ZMQProgressEmitter, + LoggingEmitter, +) + +# Public API - Exceptions +from .exceptions import ( + ProgressError, + ProgressValidationError, + ProgressRegistrationError, +) + +# ============================================================================= +# Convenience Functions +# ============================================================================= + +import time +import os +_progress_queue = None + + +def set_progress_queue(queue): + """Set progress queue for worker processes (called once at init). + + Args: + queue: multiprocessing.Queue to put progress events into + """ + global _progress_queue + _progress_queue = queue + + +def emit(**kwargs) -> None: + """Emit progress event (replaces emit_progress()). + + Invariant emission path: + 1. Validate inputs + 2. Build immutable ProgressEvent + 3. Put serialized event onto configured progress queue + + Args: + execution_id (str): Execution identifier (required) + plate_id (str): Plate identifier (required) + axis_id (str): Axis/well identifier (required) + step_name (str): Step name (required) + phase (ProgressPhase): Progress phase (required) + status (ProgressStatus): Progress status (required) + percent (float): Progress percentage 0-100 (required) + completed (int): Number of completed items (default: 0) + total (int): Total number of items (default: 1) + **kwargs: Additional metadata fields (optional) + + Raises: + ValueError: If required fields missing or invalid + ProgressError: If progress queue is not configured + """ + # Validate required fields + required_fields = { + "execution_id", + "plate_id", + "axis_id", + "step_name", + "phase", + "status", + "percent", + } + missing = required_fields - set(kwargs.keys()) + if missing: + raise ValueError(f"Missing required fields: {missing}") + + # Set defaults for optional fields + if "completed" not in kwargs: + kwargs["completed"] = 0 + if "total" not in kwargs: + kwargs["total"] = 1 + + if _progress_queue is None: + raise ProgressError( + "emit() requires explicit progress queue configuration via " + "set_progress_queue(queue). No fallback path is allowed." + ) + + event = ProgressEvent(timestamp=time.time(), pid=os.getpid(), **kwargs) + _progress_queue.put(event.to_dict()) + + +def get_registry(): + """Get process-local progress registry. + + Alias for registry() function (clearer naming). + + Returns: + ProgressRegistry singleton instance + """ + from .registry import registry + + return registry() + + +__all__ = [ + # Version + "__version__", + # Convenience + "emit", + "get_registry", +] diff --git a/openhcs/core/progress/emitters.py b/openhcs/core/progress/emitters.py new file mode 100644 index 000000000..66ecfdef4 --- /dev/null +++ b/openhcs/core/progress/emitters.py @@ -0,0 +1,167 @@ +"""Progress emitter interface and implementations.""" + +import logging +from abc import ABC, abstractmethod +from typing import Callable +import json + +from .types import ProgressEvent + +logger = logging.getLogger(__name__) + + +# ============================================================================= +# ProgressEmitter Interface (ABC) +# ============================================================================= + + +class ProgressEmitter(ABC): + """Abstract base class for progress emitters. + + Follows OpenHCS pattern: explicit ABC with single abstract method. + Emitter implementations decide WHERE progress goes (callback, ZMQ, noop). + + Design: + - ABC enforces implementation (no duck typing) + - Single abstract method (minimal surface area) + - Subclasses handle transport details + """ + + @abstractmethod + def emit(self, event: ProgressEvent) -> None: + """Emit a progress event. + + Args: + event: ProgressEvent to emit + + Raises: + ProgressError: If emission fails + """ + pass + + +# ============================================================================= +# NoopEmitter - For Testing/Disabled Progress +# ============================================================================= + + +class NoopEmitter(ProgressEmitter): + """No-op emitter for testing or disabled progress tracking. + + Does nothing on emit() - useful for: + - Unit tests (no side effects) + - Production runs with progress disabled + - Dry-run modes + """ + + def emit(self, event: ProgressEvent) -> None: + """Do nothing (intentionally no-op).""" + pass + + +# ============================================================================= +# CallbackEmitter - For In-Process Use +# ============================================================================= + + +class CallbackEmitter(ProgressEmitter): + """Callback-based emitter for in-process use. + + Useful when: + - No ZMQ required (same process) + - Direct UI updates needed + - Testing with mock callbacks + """ + + def __init__(self, callback: Callable[[ProgressEvent], None]): + """Initialize with callback. + + Args: + callback: Function to call on each emit + """ + self.callback = callback + + def emit(self, event: ProgressEvent) -> None: + """Call callback with event. + + Wraps in try/except to prevent progress failures from + crashing the pipeline (fail-safe). + """ + try: + self.callback(event) + except Exception as e: + logger.exception(f"Progress callback failed: {e}") + # Don't re-raise - progress failure shouldn't stop pipeline + + +# ============================================================================= +# ZMQProgressEmitter - For Cross-Process Communication +# ============================================================================= + + +class ZMQProgressEmitter(ProgressEmitter): + """ZMQ-based emitter for cross-process communication. + + Serializes ProgressEvent to JSON and sends via ZMQ socket. + Used for worker processes to communicate progress to main process. + """ + + def __init__(self, socket): + """Initialize with ZMQ socket. + + Args: + socket: ZMQ socket (REQ/REP pair) or None + """ + self.socket = socket + + def emit(self, event: ProgressEvent) -> None: + """Emit event via ZMQ socket. + + Serializes to JSON dict for ZMQ transport. + Handles closed socket gracefully (no crash). + + Args: + event: ProgressEvent to emit + """ + if self.socket is None: + logger.debug(f"Progress emit (no socket): {event}") + return + + try: + # Serialize to dict (enums → strings for JSON) + message = event.to_dict() + json_str = json.dumps(message) + self.socket.send_string(json_str) + except Exception as e: + logger.exception(f"ZMQ emit failed: {e}") + # Don't re-raise - progress failure shouldn't stop pipeline + + +# ============================================================================= +# LoggingEmitter - For Debugging +# ============================================================================= + + +class LoggingEmitter(ProgressEmitter): + """Logging emitter for debugging. + + Logs every progress event at INFO level. + Useful for development and troubleshooting. + """ + + def __init__(self, prefix: str = "PROGRESS"): + """Initialize with log prefix. + + Args: + prefix: Prefix for log messages (e.g., "PROGRESS") + """ + self.prefix = prefix + + def emit(self, event: ProgressEvent) -> None: + """Log event at INFO level.""" + logger.info( + f"[{self.prefix}] {event.execution_id} | " + f"{event.phase.value} | {event.status.value} | " + f"{event.step_name} | {event.axis_id} | " + f"{event.percent:.1f}% ({event.completed}/{event.total})" + ) diff --git a/openhcs/core/progress/exceptions.py b/openhcs/core/progress/exceptions.py new file mode 100644 index 000000000..c47a04bde --- /dev/null +++ b/openhcs/core/progress/exceptions.py @@ -0,0 +1,26 @@ +"""Progress-specific exceptions.""" + +from typing import Optional + + +class ProgressError(Exception): + """Base exception for progress system.""" + + pass + + +class ProgressValidationError(ProgressError): + """Raised when ProgressEvent validation fails.""" + + def __init__(self, message: str, event: dict): + self.message = message + self.event = event + super().__init__(f"{message}: {event}") + + +class ProgressRegistrationError(ProgressError): + """Raised when registry operation fails.""" + + def __init__(self, message: str, execution_id: Optional[str] = None): + self.execution_id = execution_id + super().__init__(message) diff --git a/openhcs/core/progress/projection.py b/openhcs/core/progress/projection.py new file mode 100644 index 000000000..59bc0af27 --- /dev/null +++ b/openhcs/core/progress/projection.py @@ -0,0 +1,194 @@ +"""OpenHCS runtime projection built on generic zmqruntime projection primitives.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Dict, Iterable, List, Mapping, Optional, Tuple + +from zmqruntime.progress import ( + GenericExecutionProjection, + ProgressProjectionAdapterABC, + build_execution_projection, +) + +from .types import ( + ProgressEvent, + phase_channel, + is_failure_event, + is_success_terminal_event, +) + + +class PlateRuntimeState(str, Enum): + IDLE = "idle" + COMPILING = "compiling" + COMPILED = "compiled" + EXECUTING = "executing" + COMPLETE = "complete" + FAILED = "failed" + + +@dataclass(frozen=True) +class AxisRuntimeProjection: + axis_id: str + percent: float + step_name: str + is_complete: bool + is_failed: bool + + +@dataclass(frozen=True) +class PlateRuntimeProjection: + execution_id: str + plate_id: str + state: PlateRuntimeState + percent: float + axis_progress: Tuple[AxisRuntimeProjection, ...] + latest_timestamp: float + + @property + def active_axes(self) -> Tuple[AxisRuntimeProjection, ...]: + return tuple( + axis + for axis in self.axis_progress + if not axis.is_complete and not axis.is_failed + ) + + +@dataclass +class ExecutionRuntimeProjection: + plates: List[PlateRuntimeProjection] = field(default_factory=list) + by_key: Dict[Tuple[str, str], PlateRuntimeProjection] = field(default_factory=dict) + by_plate_latest: Dict[str, PlateRuntimeProjection] = field(default_factory=dict) + compiling_count: int = 0 + compiled_count: int = 0 + executing_count: int = 0 + complete_count: int = 0 + failed_count: int = 0 + overall_percent: float = 0.0 + + def get_plate( + self, plate_id: str, execution_id: Optional[str] = None + ) -> Optional[PlateRuntimeProjection]: + if execution_id is not None: + return self.by_key.get((execution_id, plate_id)) + return self.by_plate_latest.get(plate_id) + + +class _OpenHCSProjectionAdapter( + ProgressProjectionAdapterABC[ProgressEvent, PlateRuntimeState] +): + def plate_id(self, event: ProgressEvent) -> str: + return event.plate_id + + def axis_id(self, event: ProgressEvent) -> str: + return event.axis_id + + def step_name(self, event: ProgressEvent) -> str: + return event.step_name + + def percent(self, event: ProgressEvent) -> float: + return event.percent + + def timestamp(self, event: ProgressEvent) -> float: + return event.timestamp + + def channel(self, event: ProgressEvent) -> str: + return phase_channel(event.phase).value + + def known_axes(self, events: Iterable[ProgressEvent]) -> List[str]: + axes: set[str] = set() + for event in events: + if event.total_wells: + axes.update(event.total_wells) + return sorted(axes) + + def is_failure_event(self, event: ProgressEvent) -> bool: + return is_failure_event(event) + + def is_success_terminal_event(self, event: ProgressEvent) -> bool: + return is_success_terminal_event(event) + + def state_idle(self) -> PlateRuntimeState: + return PlateRuntimeState.IDLE + + def state_compiling(self) -> PlateRuntimeState: + return PlateRuntimeState.COMPILING + + def state_compiled(self) -> PlateRuntimeState: + return PlateRuntimeState.COMPILED + + def state_executing(self) -> PlateRuntimeState: + return PlateRuntimeState.EXECUTING + + def state_complete(self) -> PlateRuntimeState: + return PlateRuntimeState.COMPLETE + + def state_failed(self) -> PlateRuntimeState: + return PlateRuntimeState.FAILED + + +_PROJECTION_ADAPTER = _OpenHCSProjectionAdapter() + + +def _from_generic_projection( + generic_projection: GenericExecutionProjection[PlateRuntimeState], +) -> ExecutionRuntimeProjection: + projection = ExecutionRuntimeProjection() + + for generic_plate in generic_projection.plates: + axis_progress = tuple( + AxisRuntimeProjection( + axis_id=axis.axis_id, + percent=axis.percent, + step_name=axis.step_name, + is_complete=axis.is_complete, + is_failed=axis.is_failed, + ) + for axis in generic_plate.axis_progress + ) + plate_projection = PlateRuntimeProjection( + execution_id=generic_plate.execution_id, + plate_id=generic_plate.plate_id, + state=generic_plate.state, + percent=generic_plate.percent, + axis_progress=axis_progress, + latest_timestamp=generic_plate.latest_timestamp, + ) + projection.plates.append(plate_projection) + projection.by_key[(plate_projection.execution_id, plate_projection.plate_id)] = ( + plate_projection + ) + + for plate_id, generic_plate in generic_projection.by_plate_latest.items(): + projection.by_plate_latest[plate_id] = projection.by_key[ + (generic_plate.execution_id, generic_plate.plate_id) + ] + + projection.compiling_count = generic_projection.count_state(PlateRuntimeState.COMPILING) + projection.compiled_count = generic_projection.count_state(PlateRuntimeState.COMPILED) + projection.executing_count = generic_projection.count_state(PlateRuntimeState.EXECUTING) + projection.complete_count = generic_projection.count_state(PlateRuntimeState.COMPLETE) + projection.failed_count = generic_projection.count_state(PlateRuntimeState.FAILED) + projection.overall_percent = generic_projection.overall_percent + + return projection + + +def build_execution_runtime_projection( + events_by_execution: Mapping[str, List[ProgressEvent]], +) -> ExecutionRuntimeProjection: + generic_projection = build_execution_projection( + events_by_execution, + adapter=_PROJECTION_ADAPTER, + ) + return _from_generic_projection(generic_projection) + + +def build_execution_runtime_projection_from_registry(progress_registry) -> ExecutionRuntimeProjection: + events_by_execution = { + execution_id: progress_registry.get_events(execution_id) + for execution_id in progress_registry.get_execution_ids() + } + return build_execution_runtime_projection(events_by_execution) diff --git a/openhcs/core/progress/registry.py b/openhcs/core/progress/registry.py new file mode 100644 index 000000000..e8821373d --- /dev/null +++ b/openhcs/core/progress/registry.py @@ -0,0 +1,83 @@ +"""OpenHCS progress registry backed by generic zmqruntime registry primitives.""" + +from __future__ import annotations + +import threading +from typing import Optional + +from zmqruntime.progress import LatestEventRegistry + +from .types import ProgressEvent, is_terminal_event, phase_channel + + +class ProgressRegistry: + """Per-process singleton registry for OpenHCS progress events.""" + + _instance: "ProgressRegistry | None" = None + _instance_lock = threading.Lock() + + def __new__(cls) -> "ProgressRegistry": + if cls._instance is None: + with cls._instance_lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self) -> None: + if self._initialized: + return + + self._retention_seconds = 60.0 + self._event_registry: LatestEventRegistry[ProgressEvent, tuple[str, str, str]] = ( + LatestEventRegistry( + key_builder=self._event_key, + is_terminal=is_terminal_event, + timestamp_of=lambda event: event.timestamp, + retention_seconds=self._retention_seconds, + ) + ) + self._initialized = True + + @staticmethod + def _event_key(event: ProgressEvent) -> tuple[str, str, str]: + channel = phase_channel(event.phase).value + return (event.plate_id, event.axis_id, channel) + + def register_event(self, execution_id: str, event: ProgressEvent) -> None: + self._event_registry.register_event(execution_id, event) + + def get_events(self, execution_id: str) -> list[ProgressEvent]: + return self._event_registry.get_events(execution_id) + + def get_latest_event(self, execution_id: str) -> Optional[ProgressEvent]: + return self._event_registry.get_latest_event(execution_id) + + def add_listener(self, listener) -> None: + self._event_registry.add_listener(listener) + + def remove_listener(self, listener) -> bool: + return self._event_registry.remove_listener(listener) + + def clear_listeners(self) -> None: + self._event_registry.clear_listeners() + + def clear_execution(self, execution_id: str) -> None: + self._event_registry.clear_execution(execution_id) + + def clear_all(self) -> None: + self._event_registry.clear_all() + + def cleanup_old_executions(self, retention_seconds: Optional[float] = None) -> int: + return self._event_registry.cleanup_old_executions(retention_seconds) + + def get_execution_ids(self) -> list[str]: + return self._event_registry.get_execution_ids() + + def get_event_count(self, execution_id: str) -> int: + return self._event_registry.get_event_count(execution_id) + + +def registry() -> ProgressRegistry: + """Get process-local progress registry (singleton).""" + return ProgressRegistry() diff --git a/openhcs/core/progress/types.py b/openhcs/core/progress/types.py new file mode 100644 index 000000000..34603ebbf --- /dev/null +++ b/openhcs/core/progress/types.py @@ -0,0 +1,478 @@ +"""Immutable progress types following OpenHCS patterns.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, replace +from enum import Enum +from typing import Dict, Any, Optional, List +import time +from zmqruntime.messages import TaskProgress + +# ============================================================================= +# ProgressPhase Enum - Unifies TaskPhase + AxisPhase +# ============================================================================= + + +class ProgressPhase(Enum): + """Progress phases - unified phase vocabulary. + + Extends ZMQRuntime's TaskPhase with OpenHCS-specific phases. + """ + + # Generic phases (from TaskPhase) + INIT = "init" + QUEUED = "queued" + RUNNING = "running" + SUCCESS = "success" + FAILED = "failed" + CANCELLED = "cancelled" + + # Compilation phases + COMPILE = "compile" + + # Execution phases + AXIS_STARTED = "axis_started" + STEP_STARTED = "step_started" + STEP_COMPLETED = "step_completed" + PATTERN_GROUP = "pattern_group" + AXIS_COMPLETED = "axis_completed" + + # Error phases + AXIS_ERROR = "axis_error" + + def __str__(self): + """String representation for logging.""" + return self.value + + +class ProgressChannel(Enum): + """Semantic channel for phase-specific progress streams.""" + + INIT = "init" + COMPILE = "compile" + PIPELINE = "pipeline" + STEP = "step" + + def __str__(self): + return self.value + + +# ============================================================================= +# ProgressStatus Enum - Unifies TaskStatus + AxisStatus +# ============================================================================= + + +class ProgressStatus(Enum): + """Progress status - unified status vocabulary. + + Extends ZMQRuntime's TaskStatus with OpenHCS-specific statuses. + """ + + # Generic statuses (from TaskStatus) + PENDING = "pending" + STARTED = "started" + RUNNING = "running" + SUCCESS = "success" + FAILED = "failed" + CANCELLED = "cancelled" + + # OpenHCS-specific statuses + ERROR = "error" + QUEUED = "queued" + + def __str__(self): + """String representation for logging.""" + return self.value + + +class ProgressSemanticsABC(ABC): + """Nominal contract for progress phase semantics.""" + + @abstractmethod + def channel_for_phase(self, phase: ProgressPhase) -> ProgressChannel: + """Classify phase into a semantic channel.""" + + @abstractmethod + def is_terminal(self, event: "ProgressEvent") -> bool: + """Return True when event is terminal.""" + + @abstractmethod + def is_execution_phase(self, phase: ProgressPhase) -> bool: + """Return True when phase belongs to execution.""" + + +class ProgressSemantics(ProgressSemanticsABC): + """Single source of truth for phase semantics.""" + + _PHASE_TO_CHANNEL = { + ProgressPhase.INIT: ProgressChannel.INIT, + ProgressPhase.QUEUED: ProgressChannel.PIPELINE, + ProgressPhase.RUNNING: ProgressChannel.PIPELINE, + ProgressPhase.SUCCESS: ProgressChannel.PIPELINE, + ProgressPhase.FAILED: ProgressChannel.PIPELINE, + ProgressPhase.CANCELLED: ProgressChannel.PIPELINE, + ProgressPhase.COMPILE: ProgressChannel.COMPILE, + ProgressPhase.AXIS_STARTED: ProgressChannel.PIPELINE, + ProgressPhase.STEP_STARTED: ProgressChannel.PIPELINE, + ProgressPhase.STEP_COMPLETED: ProgressChannel.PIPELINE, + ProgressPhase.PATTERN_GROUP: ProgressChannel.STEP, + ProgressPhase.AXIS_COMPLETED: ProgressChannel.PIPELINE, + ProgressPhase.AXIS_ERROR: ProgressChannel.PIPELINE, + } + _TERMINAL_PHASES = { + ProgressPhase.SUCCESS, + ProgressPhase.FAILED, + ProgressPhase.CANCELLED, + ProgressPhase.AXIS_COMPLETED, + ProgressPhase.AXIS_ERROR, + } + _TERMINAL_STATUSES = { + ProgressStatus.SUCCESS, + ProgressStatus.FAILED, + ProgressStatus.CANCELLED, + ProgressStatus.ERROR, + } + + def channel_for_phase(self, phase: ProgressPhase) -> ProgressChannel: + return self._PHASE_TO_CHANNEL[phase] + + def is_terminal(self, event: "ProgressEvent") -> bool: + return ( + event.phase in self._TERMINAL_PHASES + or event.status in self._TERMINAL_STATUSES + ) + + def is_execution_phase(self, phase: ProgressPhase) -> bool: + channel = self.channel_for_phase(phase) + return channel in {ProgressChannel.PIPELINE, ProgressChannel.STEP} + + +_PROGRESS_SEMANTICS = ProgressSemantics() +_FAILURE_STATUSES = { + ProgressStatus.FAILED, + ProgressStatus.ERROR, + ProgressStatus.CANCELLED, +} +_FAILURE_PHASES = { + ProgressPhase.FAILED, + ProgressPhase.CANCELLED, + ProgressPhase.AXIS_ERROR, +} +_SUCCESS_TERMINAL_PHASES = { + ProgressPhase.SUCCESS, + ProgressPhase.AXIS_COMPLETED, +} + + +def phase_channel(phase: ProgressPhase) -> ProgressChannel: + """Classify phase to semantic channel.""" + return _PROGRESS_SEMANTICS.channel_for_phase(phase) + + +def is_terminal_event(event: "ProgressEvent") -> bool: + """True when the event is terminal.""" + return _PROGRESS_SEMANTICS.is_terminal(event) + + +def is_execution_phase(phase: ProgressPhase) -> bool: + """True when phase belongs to execution tree.""" + return _PROGRESS_SEMANTICS.is_execution_phase(phase) + + +def is_failure_event(event: "ProgressEvent") -> bool: + """True when event represents a failure state.""" + return event.status in _FAILURE_STATUSES or event.phase in _FAILURE_PHASES + + +def is_success_terminal_event(event: "ProgressEvent") -> bool: + """True when event represents successful terminal completion.""" + return event.phase in _SUCCESS_TERMINAL_PHASES + + +# ============================================================================= +# ProgressEvent Frozen Dataclass - Single Source of Truth +# ============================================================================= + + +@dataclass(frozen=True) +class ProgressEvent: + """Immutable progress event - single source of truth. + + Replaces dict-based progress payloads with validated, immutable data. + Uses frozen=True to ensure thread-safety and prevent accidental mutation. + + All fields are explicit and typed - no generic metadata dict. + """ + + # Required core identifiers + execution_id: str + plate_id: str + axis_id: str + step_name: str + + # Progress tracking + phase: ProgressPhase + status: ProgressStatus + percent: float + completed: int + total: int + + # Metadata (timestamp, PID) + timestamp: float + pid: int + + # Optional error information + error: Optional[str] = None + traceback: Optional[str] = None + + # Optional application-specific fields + total_wells: Optional[List[str]] = None + worker_assignments: Optional[Dict[str, List[str]]] = None + worker_slot: Optional[str] = None + owned_wells: Optional[List[str]] = None + message: Optional[str] = None # General message field (e.g., error messages) + component: Optional[str] = None # Component value for pattern group progress + pattern: Optional[str] = None # Pattern value for pattern group progress + context: Optional[Dict[str, Any]] = None # Generic context for arbitrary data + step_names: Optional[List[str]] = None # Step names for the pipeline + + def __post_init__(self): + """Validate invariants (fail-loud principle).""" + # Validate percent range + if not (0.0 <= self.percent <= 100.0): + raise ValueError(f"percent must be in [0.0, 100.0], got {self.percent}") + + # Validate completed <= total + if self.completed > self.total: + raise ValueError( + f"completed ({self.completed}) cannot exceed total ({self.total})" + ) + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ProgressEvent": + """Create ProgressEvent from dict (for ZMQ transport). + + Converts string phase/status to enums for type safety. + + Args: + data: Dictionary with progress data (from ZMQ message) + + Returns: + ProgressEvent instance + + Raises: + KeyError: If required fields missing + ValueError: If phase/status strings invalid + TypeError: If field types invalid + """ + # Validate generic transport invariants using zmqruntime primitive + TaskProgress.from_dict(data) + + # Validate OpenHCS-required fields + required_fields = { + "execution_id", + "plate_id", + "axis_id", + "step_name", + "phase", + "status", + "percent", + "completed", + "total", + "timestamp", + "pid", + } + missing = required_fields - set(data.keys()) + if missing: + raise KeyError( + f"Missing required fields: {missing}. Got keys: {list(data.keys())}" + ) + + # Convert phase string to enum + phase_str = data["phase"] + try: + phase = ProgressPhase(phase_str) + except ValueError: + raise ValueError( + f"Invalid phase '{phase_str}'. Valid phases: " + f"{[p.value for p in ProgressPhase]}" + ) + + # Convert status string to enum + status_str = data["status"] + try: + status = ProgressStatus(status_str) + except ValueError: + raise ValueError( + f"Invalid status '{status_str}'. Valid statuses: " + f"{[s.value for s in ProgressStatus]}" + ) + + # Create event with all fields (optional fields use .get()) + return cls( + execution_id=data["execution_id"], + plate_id=data["plate_id"], + axis_id=data["axis_id"], + step_name=data["step_name"], + phase=phase, + status=status, + percent=float(data["percent"]), + completed=int(data["completed"]), + total=int(data["total"]), + timestamp=float(data["timestamp"]), + pid=int(data["pid"]), + error=data.get("error"), + traceback=data.get("traceback"), + total_wells=data.get("total_wells"), + worker_assignments=data.get("worker_assignments"), + worker_slot=data.get("worker_slot"), + owned_wells=data.get("owned_wells"), + message=data.get("message"), + component=data.get("component"), + pattern=data.get("pattern"), + context=data.get("context"), + step_names=data.get("step_names"), + ) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dict (for ZMQ transport). + + Converts enums to strings for JSON serialization. + Only includes optional fields if they are not None. + + Returns: + Dictionary representation of this event + """ + result = { + "execution_id": str(self.execution_id), + "plate_id": str(self.plate_id), + "axis_id": str(self.axis_id), + "step_name": self.step_name, + "phase": self.phase.value, # Enum → string + "status": self.status.value, # Enum → string + "percent": self.percent, + "completed": self.completed, + "total": self.total, + "timestamp": self.timestamp, + "pid": self.pid, + } + + # Add optional fields if present + if self.error is not None: + result["error"] = self.error + if self.traceback is not None: + result["traceback"] = self.traceback + if self.total_wells is not None: + result["total_wells"] = self.total_wells + if self.worker_assignments is not None: + result["worker_assignments"] = self.worker_assignments + if self.worker_slot is not None: + result["worker_slot"] = self.worker_slot + if self.owned_wells is not None: + result["owned_wells"] = self.owned_wells + if self.message is not None: + result["message"] = self.message + if self.component is not None: + result["component"] = self.component + if self.pattern is not None: + result["pattern"] = self.pattern + if self.context is not None: + result["context"] = self.context + if self.step_names is not None: + result["step_names"] = self.step_names + + return result + + def replace(self, **kwargs) -> "ProgressEvent": + """Create a copy with replaced fields (immutable update pattern). + + Returns: + New ProgressEvent with specified fields replaced + """ + return replace(self, **kwargs) + + def is_complete(self) -> bool: + """Check if this event represents a completed/terminal state. + + Returns: + True if the event is in a terminal phase or status + """ + return is_terminal_event(self) + + +# ============================================================================= +# Utility Functions +# ============================================================================= + + +def create_event( + execution_id: str, + plate_id: str, + axis_id: str, + step_name: str, + phase: ProgressPhase, + status: ProgressStatus, + percent: float, + completed: int = 0, + total: int = 1, + error: Optional[str] = None, + traceback: Optional[str] = None, + total_wells: Optional[List[str]] = None, + worker_assignments: Optional[Dict[str, List[str]]] = None, + worker_slot: Optional[str] = None, + owned_wells: Optional[List[str]] = None, + message: Optional[str] = None, + component: Optional[str] = None, + pattern: Optional[str] = None, +) -> ProgressEvent: + """Convenience function to create ProgressEvent with defaults. + + Automatically sets timestamp and pid for caller. + + Args: + execution_id: Execution identifier + plate_id: Plate identifier + axis_id: Axis/well identifier + step_name: Name of current step + phase: Progress phase enum + status: Progress status enum + percent: Progress percentage (0-100) + completed: Number of completed items + total: Total number of items + error: Optional error message + traceback: Optional error traceback + total_wells: Optional list of well identifiers + worker_assignments: Optional worker->well map + worker_slot: Optional worker slot ID for the emitting worker + owned_wells: Optional owned well list for the emitting worker + message: Optional general message + component: Optional component value for pattern group progress + pattern: Optional pattern value for pattern group progress + + Returns: + ProgressEvent instance with timestamp and pid set + + Raises: + ValueError: If validation fails + """ + return ProgressEvent( + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name=step_name, + phase=phase, + status=status, + percent=percent, + completed=completed, + total=total, + timestamp=time.time(), + pid=__import__("os").getpid(), + error=error, + traceback=traceback, + total_wells=total_wells, + worker_assignments=worker_assignments, + worker_slot=worker_slot, + owned_wells=owned_wells, + message=message, + component=component, + pattern=pattern, + ) diff --git a/openhcs/core/steps/function_step.py b/openhcs/core/steps/function_step.py index 587c0e855..5a5dfc372 100644 --- a/openhcs/core/steps/function_step.py +++ b/openhcs/core/steps/function_step.py @@ -11,17 +11,30 @@ import os import time from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Tuple, Union, OrderedDict as TypingOrderedDict, TYPE_CHECKING +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Tuple, + Union, + OrderedDict as TypingOrderedDict, + TYPE_CHECKING, +) if TYPE_CHECKING: pass -from openhcs.constants.constants import (DEFAULT_IMAGE_EXTENSIONS, - Backend, - VariableComponents) +from openhcs.constants.constants import ( + DEFAULT_IMAGE_EXTENSIONS, + Backend, + VariableComponents, +) from openhcs.core.context.processing_context import ProcessingContext from openhcs.core.steps.abstract import AbstractStep +from openhcs.core.progress import emit, ProgressPhase, ProgressStatus from openhcs.formats.func_arg_prep import prepare_patterns_and_functions from openhcs.core.memory import stack_slices, unstack_slices # OpenHCS imports moved to local imports to avoid circular dependencies @@ -29,7 +42,10 @@ logger = logging.getLogger(__name__) -def _generate_materialized_paths(memory_paths: List[str], step_output_dir: Path, materialized_output_dir: Path) -> List[str]: + +def _generate_materialized_paths( + memory_paths: List[str], step_output_dir: Path, materialized_output_dir: Path +) -> List[str]: """Generate materialized file paths by replacing step output directory.""" materialized_paths = [] for memory_path in memory_paths: @@ -40,8 +56,7 @@ def _generate_materialized_paths(memory_paths: List[str], step_output_dir: Path, def _filter_special_outputs_for_function( - outputs_to_save: List[str], - special_outputs_map: Dict + outputs_to_save: List[str], special_outputs_map: Dict ) -> Dict: """Filter special outputs for a specific function call. @@ -61,9 +76,7 @@ def _filter_special_outputs_for_function( def _select_special_plan_for_component( - plan_by_group: Optional[Dict], - component_key: Optional[str], - default_plan: Dict + plan_by_group: Optional[Dict], component_key: Optional[str], default_plan: Dict ) -> Dict: """Select precompiled special I/O plan for a component.""" if not plan_by_group: @@ -77,10 +90,7 @@ def _select_special_plan_for_component( def _filter_patterns_by_component( - patterns: Union[List, Dict], - component: str, - target_value: str, - microscope_handler + patterns: Union[List, Dict], component: str, target_value: str, microscope_handler ) -> Union[List, Dict]: """Filter patterns to only include those matching a specific component value. @@ -105,7 +115,9 @@ def filter_pattern_list(pattern_list: List) -> List: for pattern in pattern_list: # Replace placeholder with dummy value to make pattern parseable # This follows the same convention as PatternDiscoveryEngine - pattern_template = str(pattern).replace(PatternDiscoveryEngine.PLACEHOLDER_PATTERN, '001') + pattern_template = str(pattern).replace( + PatternDiscoveryEngine.PLACEHOLDER_PATTERN, "001" + ) metadata = microscope_handler.parser.parse_filename(pattern_template) if metadata and str(metadata.get(component)) == str(target_value): @@ -126,32 +138,45 @@ def filter_pattern_list(pattern_list: List) -> List: return filter_pattern_list(patterns) -def _save_materialized_data(filemanager, memory_data: List, materialized_paths: List[str], - materialized_backend: str, step_plan: Dict, context, axis_id: str) -> None: +def _save_materialized_data( + filemanager, + memory_data: List, + materialized_paths: List[str], + materialized_backend: str, + step_plan: Dict, + context, + axis_id: str, +) -> None: """Save data to materialized location using appropriate backend.""" # Build kwargs with parser metadata (all backends receive it) save_kwargs = { - 'parser_name': context.microscope_handler.parser.__class__.__name__, - 'microscope_type': context.microscope_handler.microscope_type + "parser_name": context.microscope_handler.parser.__class__.__name__, + "microscope_type": context.microscope_handler.microscope_type, } if materialized_backend == Backend.ZARR.value: - n_channels, n_z, n_fields = _calculate_zarr_dimensions(materialized_paths, context.microscope_handler) - row, col = context.microscope_handler.parser.extract_component_coordinates(axis_id) - save_kwargs.update({ - 'chunk_name': axis_id, - 'zarr_config': step_plan.get("zarr_config"), - 'n_channels': n_channels, - 'n_z': n_z, - 'n_fields': n_fields, - 'row': row, - 'col': col - }) - - filemanager.save_batch(memory_data, materialized_paths, materialized_backend, **save_kwargs) - + n_channels, n_z, n_fields = _calculate_zarr_dimensions( + materialized_paths, context.microscope_handler + ) + row, col = context.microscope_handler.parser.extract_component_coordinates( + axis_id + ) + save_kwargs.update( + { + "chunk_name": axis_id, + "zarr_config": step_plan.get("zarr_config"), + "n_channels": n_channels, + "n_z": n_z, + "n_fields": n_fields, + "row": row, + "col": col, + } + ) + filemanager.save_batch( + memory_data, materialized_paths, materialized_backend, **save_kwargs + ) def get_all_image_paths(input_dir, backend, axis_id, filemanager, microscope_handler): @@ -180,6 +205,7 @@ def get_all_image_paths(input_dir, backend, axis_id, filemanager, microscope_han metadata = parser.parse_filename(filename) # Use dynamic multiprocessing axis instead of hardcoded 'well' from openhcs.constants import MULTIPROCESSING_AXIS + axis_key = MULTIPROCESSING_AXIS.value if metadata and metadata.get(axis_key) == axis_id: axis_files.append(str(f)) @@ -191,7 +217,9 @@ def get_all_image_paths(input_dir, backend, axis_id, filemanager, microscope_han input_dir_path = Path(input_dir) full_file_paths = [str(input_dir_path / Path(f).name) for f in sorted_files] - logger.debug(f"Found {len(all_image_files)} total files, {len(full_file_paths)} for axis {axis_id}") + logger.debug( + f"Found {len(all_image_files)} total files, {len(full_file_paths)} for axis {axis_id}" + ) return full_file_paths @@ -208,18 +236,22 @@ def create_image_path_getter(axis_id, filemanager, microscope_handler): Returns: Function that takes (input_dir, backend) and returns image paths for the well """ + def get_paths_for_axis(input_dir, backend): return get_all_image_paths( input_dir=input_dir, axis_id=axis_id, backend=backend, filemanager=filemanager, - microscope_handler=microscope_handler + microscope_handler=microscope_handler, ) + return get_paths_for_axis + # Environment variable to disable universal GPU defragmentation -DISABLE_GPU_DEFRAG = os.getenv('OPENHCS_DISABLE_GPU_DEFRAG', 'false').lower() == 'true' +DISABLE_GPU_DEFRAG = os.getenv("OPENHCS_DISABLE_GPU_DEFRAG", "false").lower() == "true" + def _bulk_preload_step_images( step_input_dir: Path, @@ -227,11 +259,11 @@ def _bulk_preload_step_images( axis_id: str, read_backend: str, patterns_by_well: Dict[str, Any], - filemanager: 'FileManager', - microscope_handler: 'MicroscopeHandler', + filemanager: "FileManager", + microscope_handler: "MicroscopeHandler", zarr_config: Optional[Dict[str, Any]] = None, patterns_to_preload: Optional[List[str]] = None, - variable_components: Optional[List[str]] = None + variable_components: Optional[List[str]] = None, ) -> None: """ Pre-load images for this step from source backend into memory backend. @@ -246,28 +278,41 @@ def _bulk_preload_step_images( Note: External conditional logic ensures this is only called for non-memory backends. """ import time - start_time = time.time() + start_time = time.time() # Get file paths based on mode if patterns_to_preload is not None: # Sequential mode: expand patterns to files - all_files = [f for p in patterns_to_preload - for f in microscope_handler.path_list_from_pattern( - str(step_input_dir), p, filemanager, read_backend, variable_components)] + all_files = [ + f + for p in patterns_to_preload + for f in microscope_handler.path_list_from_pattern( + str(step_input_dir), p, filemanager, read_backend, variable_components + ) + ] # Ensure full paths (prepend directory if needed) - full_file_paths = [str(step_input_dir / f) if not Path(f).is_absolute() else f for f in set(all_files)] + full_file_paths = [ + str(step_input_dir / f) if not Path(f).is_absolute() else f + for f in set(all_files) + ] else: # Normal mode: get all files for well - get_paths_for_axis = create_image_path_getter(axis_id, filemanager, microscope_handler) + get_paths_for_axis = create_image_path_getter( + axis_id, filemanager, microscope_handler + ) full_file_paths = get_paths_for_axis(step_input_dir, read_backend) if not full_file_paths: - raise RuntimeError(f"🔄 BULK PRELOAD: No files found for well {axis_id} in {step_input_dir} with backend {read_backend}") + raise RuntimeError( + f"🔄 BULK PRELOAD: No files found for well {axis_id} in {step_input_dir} with backend {read_backend}" + ) # Load from source backend with conditional zarr_config if read_backend == Backend.ZARR.value: - raw_images = filemanager.load_batch(full_file_paths, read_backend, zarr_config=zarr_config) + raw_images = filemanager.load_batch( + full_file_paths, read_backend, zarr_config=zarr_config + ) else: raw_images = filemanager.load_batch(full_file_paths, read_backend) @@ -275,7 +320,7 @@ def _bulk_preload_step_images( filemanager.ensure_directory(str(step_input_dir), Backend.MEMORY.value) # Save to memory backend using OUTPUT paths - # memory_paths = [str(step_output_dir / Path(fp).name) for fp in full_file_paths] + # memory_paths = [str(step_output_dir / Path(fp).name) for fp in full_file_paths] for file_path in full_file_paths: if filemanager.exists(file_path, Backend.MEMORY.value): filemanager.delete(file_path, Backend.MEMORY.value) @@ -287,13 +332,14 @@ def _bulk_preload_step_images( load_time = time.time() - start_time + def _bulk_writeout_step_images( step_output_dir: Path, write_backend: str, axis_id: str, zarr_config: Optional[Dict[str, Any]], - filemanager: 'FileManager', - microscope_handler: Optional[Any] = None + filemanager: "FileManager", + microscope_handler: Optional[Any] = None, ) -> None: """ Write all processed images from memory to final backend (disk/zarr). @@ -304,24 +350,28 @@ def _bulk_writeout_step_images( Note: External conditional logic ensures this is only called for non-memory backends. """ import time - start_time = time.time() + start_time = time.time() # Create specialized path getter and get memory paths for this well - get_paths_for_axis = create_image_path_getter(axis_id, filemanager, microscope_handler) + get_paths_for_axis = create_image_path_getter( + axis_id, filemanager, microscope_handler + ) memory_file_paths = get_paths_for_axis(step_output_dir, Backend.MEMORY.value) if not memory_file_paths: - raise RuntimeError(f"🔄 BULK WRITEOUT: No image files found for well {axis_id} in memory directory {step_output_dir}") + raise RuntimeError( + f"🔄 BULK WRITEOUT: No image files found for well {axis_id} in memory directory {step_output_dir}" + ) # Convert relative memory paths back to absolute paths for target backend # Memory backend stores relative paths, but target backend needs absolute paths -# file_paths = -# for memory_path in memory_file_paths: -# # Get just the filename and construct proper target path -# filename = Path(memory_path).name -# target_path = step_output_dir / filename -# file_paths.append(str(target_path)) + # file_paths = + # for memory_path in memory_file_paths: + # # Get just the filename and construct proper target path + # filename = Path(memory_path).name + # target_path = step_output_dir / filename + # file_paths.append(str(target_path)) file_paths = memory_file_paths @@ -335,22 +385,41 @@ def _bulk_writeout_step_images( if write_backend == Backend.ZARR.value: # Calculate zarr dimensions from file paths if microscope_handler is not None: - n_channels, n_z, n_fields = _calculate_zarr_dimensions(file_paths, microscope_handler) + n_channels, n_z, n_fields = _calculate_zarr_dimensions( + file_paths, microscope_handler + ) # Parse well to get row and column for zarr structure row, col = microscope_handler.parser.extract_component_coordinates(axis_id) - filemanager.save_batch(memory_data, file_paths, write_backend, - chunk_name=axis_id, zarr_config=zarr_config, - n_channels=n_channels, n_z=n_z, n_fields=n_fields, - row=row, col=col) + filemanager.save_batch( + memory_data, + file_paths, + write_backend, + chunk_name=axis_id, + zarr_config=zarr_config, + n_channels=n_channels, + n_z=n_z, + n_fields=n_fields, + row=row, + col=col, + ) else: # Fallback without dimensions if microscope_handler not available - filemanager.save_batch(memory_data, file_paths, write_backend, chunk_name=axis_id, zarr_config=zarr_config) + filemanager.save_batch( + memory_data, + file_paths, + write_backend, + chunk_name=axis_id, + zarr_config=zarr_config, + ) else: filemanager.save_batch(memory_data, file_paths, write_backend) write_time = time.time() - start_time -def _calculate_zarr_dimensions(file_paths: List[Union[str, Path]], microscope_handler) -> tuple[int, int, int]: + +def _calculate_zarr_dimensions( + file_paths: List[Union[str, Path]], microscope_handler +) -> tuple[int, int, int]: """ Calculate zarr dimensions (n_channels, n_z, n_fields) from file paths using microscope parser. @@ -368,9 +437,15 @@ def _calculate_zarr_dimensions(file_paths: List[Union[str, Path]], microscope_ha parsed_files.append(metadata) # Count unique values for each dimension from actual files - n_channels = len(set(f.get('channel') for f in parsed_files if f.get('channel') is not None)) - n_z = len(set(f.get('z_index') for f in parsed_files if f.get('z_index') is not None)) - n_fields = len(set(f.get('site') for f in parsed_files if f.get('site') is not None)) + n_channels = len( + set(f.get("channel") for f in parsed_files if f.get("channel") is not None) + ) + n_z = len( + set(f.get("z_index") for f in parsed_files if f.get("z_index") is not None) + ) + n_fields = len( + set(f.get("site") for f in parsed_files if f.get("site") is not None) + ) # Ensure at least 1 for each dimension (handle cases where metadata is missing) n_channels = max(1, n_channels) @@ -380,22 +455,24 @@ def _calculate_zarr_dimensions(file_paths: List[Union[str, Path]], microscope_ha return n_channels, n_z, n_fields - def _is_3d(array: Any) -> bool: """Check if an array is 3D.""" - return hasattr(array, 'ndim') and array.ndim == 3 + return hasattr(array, "ndim") and array.ndim == 3 + def _execute_function_core( func_callable: Callable, main_data_arg: Any, base_kwargs: Dict[str, Any], - context: 'ProcessingContext', + context: "ProcessingContext", special_inputs_plan: Dict[str, str], # {'arg_name_for_func': 'special_path_value'} - special_outputs_plan: TypingOrderedDict[str, str], # {'output_key': 'special_path_value'}, order matters - axis_id: str, # Add axis_id parameter + special_outputs_plan: TypingOrderedDict[ + str, str + ], # {'output_key': 'special_path_value'}, order matters + axis_id: str, # Add axis_id parameter input_memory_type: str, - device_id: int -) -> Any: # Returns the main processed data stack + device_id: int, +) -> Any: # Returns the main processed data stack """ Executes a single callable, handling its special I/O. - Loads special inputs from VFS paths in `special_inputs_plan`. @@ -408,32 +485,40 @@ def _execute_function_core( # Log dtype_config in kwargs if special_inputs_plan: - logger.info(f"�� SPECIAL_INPUTS_DEBUG : special_inputs_plan = {special_inputs_plan}") + logger.info( + f"�� SPECIAL_INPUTS_DEBUG : special_inputs_plan = {special_inputs_plan}" + ) for arg_name, path_info in special_inputs_plan.items(): - - # Extract path string from the path info dictionary # Current format: {"path": "/path/to/file.pkl", "source_step_id": "step_123"} - if isinstance(path_info, dict) and 'path' in path_info: - special_path_value = path_info['path'] + if isinstance(path_info, dict) and "path" in path_info: + special_path_value = path_info["path"] else: special_path_value = path_info # Fallback if it's already a string - logger.info(f"Loading special input '{arg_name}' from path '{special_path_value}' (memory backend)") + logger.info( + f"Loading special input '{arg_name}' from path '{special_path_value}' (memory backend)" + ) try: - final_kwargs[arg_name] = context.filemanager.load(special_path_value, Backend.MEMORY.value) + final_kwargs[arg_name] = context.filemanager.load( + special_path_value, Backend.MEMORY.value + ) except Exception as e: - logger.error(f"Failed to load special input '{arg_name}' from '{special_path_value}': {e}", exc_info=True) + logger.error( + f"Failed to load special input '{arg_name}' from '{special_path_value}': {e}", + exc_info=True, + ) raise # Auto-inject context if function signature expects it import inspect + sig = inspect.signature(func_callable) - if 'context' in sig.parameters: - final_kwargs['context'] = context + if "context" in sig.parameters: + final_kwargs["context"] = context # 🔍 DEBUG: Log input dimensions - input_shape = getattr(main_data_arg, 'shape', 'no shape attr') + input_shape = getattr(main_data_arg, "shape", "no shape attr") input_type = type(main_data_arg).__name__ # ⚡ INFO: Terse function execution log for user feedback @@ -457,54 +542,80 @@ def _execute_function_core( if special_outputs_plan: # Iterate through special_outputs_plan (which must be ordered by compiler) # and match with positionally returned special values. - for i, (output_key, vfs_path_info) in enumerate(special_outputs_plan.items()): - logger.info(f"Saving special output '{output_key}' to VFS path '{vfs_path_info}' (memory backend)") + for i, (output_key, vfs_path_info) in enumerate( + special_outputs_plan.items() + ): + logger.info( + f"Saving special output '{output_key}' to VFS path '{vfs_path_info}' (memory backend)" + ) if i < len(returned_special_values_tuple): value_to_save = returned_special_values_tuple[i] # Extract path string from the path info dictionary # Current format: {"path": "/path/to/file.pkl"} - if isinstance(vfs_path_info, dict) and 'path' in vfs_path_info: - vfs_path = vfs_path_info['path'] + if isinstance(vfs_path_info, dict) and "path" in vfs_path_info: + vfs_path = vfs_path_info["path"] else: vfs_path = vfs_path_info # Fallback if it's already a string # DEBUG: List what's currently in VFS before saving - from polystore.base import storage_registry as global_storage_registry - global_memory_backend = global_storage_registry[Backend.MEMORY.value] - global_existing_keys = list(global_memory_backend._memory_store.keys()) + from polystore.base import ( + storage_registry as global_storage_registry, + ) + + global_memory_backend = global_storage_registry[ + Backend.MEMORY.value + ] + global_existing_keys = list( + global_memory_backend._memory_store.keys() + ) # Check filemanager's memory backend - filemanager_memory_backend = context.filemanager._get_backend(Backend.MEMORY.value) - filemanager_existing_keys = list(filemanager_memory_backend._memory_store.keys()) + filemanager_memory_backend = context.filemanager._get_backend( + Backend.MEMORY.value + ) + filemanager_existing_keys = list( + filemanager_memory_backend._memory_store.keys() + ) if vfs_path in filemanager_existing_keys: - logger.warning(f"🔍 VFS_DEBUG: WARNING - '{vfs_path}' ALREADY EXISTS in FILEMANAGER memory backend!") + logger.warning( + f"🔍 VFS_DEBUG: WARNING - '{vfs_path}' ALREADY EXISTS in FILEMANAGER memory backend!" + ) # Ensure directory exists for memory backend parent_dir = str(Path(vfs_path).parent) - context.filemanager.ensure_directory(parent_dir, Backend.MEMORY.value) - context.filemanager.save(value_to_save, vfs_path, Backend.MEMORY.value) + context.filemanager.ensure_directory( + parent_dir, Backend.MEMORY.value + ) + context.filemanager.save( + value_to_save, vfs_path, Backend.MEMORY.value + ) else: # This indicates a mismatch that should ideally be caught by schema/validation - logger.error(f"Mismatch: special_outputs_plan wants to save '{output_key}', but function only returned {len(returned_special_values_tuple)} special values.") - raise ValueError(f"Function did not return enough values for all planned special outputs. Missing value for '{output_key}'.") + logger.error( + f"Mismatch: special_outputs_plan wants to save '{output_key}', but function only returned {len(returned_special_values_tuple)} special values." + ) + raise ValueError( + f"Function did not return enough values for all planned special outputs. Missing value for '{output_key}'." + ) else: # Function did not return a tuple - use output directly main_output_data = raw_function_output return main_output_data + def _execute_chain_core( initial_data_stack: Any, func_chain: List[Union[Callable, Tuple[Callable, Dict]]], - context: 'ProcessingContext', + context: "ProcessingContext", step_special_inputs_plan: Dict[str, str], step_special_outputs_plan: TypingOrderedDict[str, str], axis_id: str, # Add axis_id parameter device_id: int, input_memory_type: str, step_index: int, # Add step_index for funcplan lookup - dict_key: str = "default" # Add dict_key for funcplan lookup + dict_key: str = "default", # Add dict_key for funcplan lookup ) -> Any: current_stack = initial_data_stack current_memory_type = input_memory_type # Track memory type from frozen context @@ -512,10 +623,11 @@ def _execute_chain_core( for i, func_item in enumerate(func_chain): actual_callable: Callable base_kwargs_for_item: Dict[str, Any] = {} - is_last_in_chain = (i == len(func_chain) - 1) + is_last_in_chain = i == len(func_chain) - 1 # Resolve FunctionReference objects to actual functions in worker process from openhcs.core.pipeline.compiler import FunctionReference + if isinstance(func_item, FunctionReference): actual_callable = func_item.resolve() elif isinstance(func_item, tuple) and len(func_item) == 2: @@ -527,6 +639,16 @@ def _execute_chain_core( else: raise TypeError(f"Invalid function in tuple: {func_or_ref}") base_kwargs_for_item = kwargs + # Strip UI-only metadata keys (never passed to runtime callables) + if ( + isinstance(base_kwargs_for_item, dict) + and "__pyqt_reactive_scope_token__" in base_kwargs_for_item + ): + base_kwargs_for_item = { + k: v + for k, v in base_kwargs_for_item.items() + if k != "__pyqt_reactive_scope_token__" + } elif callable(func_item): actual_callable = func_item else: @@ -534,16 +656,17 @@ def _execute_chain_core( # Convert to function's input memory type (noop if same) from openhcs.core.memory import convert_memory + current_stack = convert_memory( data=current_stack, source_type=current_memory_type, target_type=actual_callable.input_memory_type, - gpu_id=device_id + gpu_id=device_id, ) # Use funcplan to determine which outputs this function should save funcplan = context.step_plans[step_index].get("funcplan", {}) - func_name = getattr(actual_callable, '__name__', 'unknown') + func_name = getattr(actual_callable, "__name__", "unknown") # Construct execution key: function_name_dict_key_chain_position execution_key = f"{func_name}_{dict_key}_{i}" @@ -574,8 +697,9 @@ def _execute_chain_core( return current_stack + def _process_single_pattern_group( - context: 'ProcessingContext', + context: "ProcessingContext", pattern_group_info: Any, executable_func_or_chain: Any, base_func_args: Dict[str, Any], @@ -585,15 +709,15 @@ def _process_single_pattern_group( component_value: str, read_backend: str, write_backend: str, - input_memory_type_from_plan: str, # Explicitly from plan - output_memory_type_from_plan: str, # Explicitly from plan + input_memory_type_from_plan: str, # Explicitly from plan + output_memory_type_from_plan: str, # Explicitly from plan device_id: Optional[int], same_directory: bool, special_inputs_map: Dict[str, str], special_outputs_map: TypingOrderedDict[str, str], zarr_config: Optional[Dict[str, Any]], variable_components: Optional[List[str]] = None, - step_index: Optional[int] = None # Add step_index for funcplan lookup + step_index: Optional[int] = None, # Add step_index for funcplan lookup ) -> None: start_time = time.time() pattern_repr = str(pattern_group_info)[:100] @@ -601,11 +725,14 @@ def _process_single_pattern_group( try: if not context.microscope_handler: - raise RuntimeError("MicroscopeHandler not available in context.") + raise RuntimeError("MicroscopeHandler not available in context.") matching_files = context.microscope_handler.path_list_from_pattern( - str(step_input_dir), pattern_group_info, context.filemanager, Backend.MEMORY.value, - [vc.value for vc in variable_components] if variable_components else None + str(step_input_dir), + pattern_group_info, + context.filemanager, + Backend.MEMORY.value, + [vc.value for vc in variable_components] if variable_components else None, ) if not matching_files: @@ -616,14 +743,20 @@ def _process_single_pattern_group( f"Check that input files exist and match the expected naming convention." ) - logger.debug(f"🔥 PATTERN: Found {len(matching_files)} files: {[Path(f).name for f in matching_files]}") + logger.debug( + f"🔥 PATTERN: Found {len(matching_files)} files: {[Path(f).name for f in matching_files]}" + ) # Sort files to ensure consistent ordering (especially important for z-stacks) matching_files.sort() - logger.debug(f"🔥 PATTERN: Sorted files: {[Path(f).name for f in matching_files]}") + logger.debug( + f"🔥 PATTERN: Sorted files: {[Path(f).name for f in matching_files]}" + ) full_file_paths = [str(step_input_dir / f) for f in matching_files] - raw_slices = context.filemanager.load_batch(full_file_paths, Backend.MEMORY.value) + raw_slices = context.filemanager.load_batch( + full_file_paths, Backend.MEMORY.value + ) if not raw_slices: raise ValueError( @@ -635,55 +768,82 @@ def _process_single_pattern_group( # 🔍 DEBUG: Log stacking operation if raw_slices: - slice_shapes = [getattr(s, 'shape', 'no shape') for s in raw_slices[:3]] # First 3 shapes + slice_shapes = [ + getattr(s, "shape", "no shape") for s in raw_slices[:3] + ] # First 3 shapes main_data_stack = stack_slices( slices=raw_slices, memory_type=input_memory_type_from_plan, gpu_id=device_id ) # 🔍 DEBUG: Log stacked result - stack_shape = getattr(main_data_stack, 'shape', 'no shape') + stack_shape = getattr(main_data_stack, "shape", "no shape") stack_type = type(main_data_stack).__name__ - final_base_kwargs = base_func_args.copy() component_key = None if component_value is None else str(component_value) # Get step function from step plan step_func = context.step_plans[step_index]["func"] + # DEBUG: Log component_key and available groups + logger.info(f"🔍 COMPONENT_KEY: component_value={component_value}, component_key={component_key}") + special_outputs_by_group = context.step_plans[step_index].get("special_outputs_by_group") + if special_outputs_by_group: + logger.info(f"🔍 AVAILABLE_GROUPS: {list(special_outputs_by_group.keys())}") + else: + logger.info(f"🔍 NO special_outputs_by_group in step plan") + if isinstance(step_func, dict): - dict_key_for_funcplan = component_key # Use actual dict key for dict patterns + dict_key_for_funcplan = ( + component_key # Use actual dict key for dict patterns + ) else: dict_key_for_funcplan = "default" # Use default for list/single patterns special_inputs_for_component = _select_special_plan_for_component( context.step_plans[step_index].get("special_inputs_by_group"), component_key, - special_inputs_map + special_inputs_map, ) special_outputs_for_component = _select_special_plan_for_component( context.step_plans[step_index].get("special_outputs_by_group"), component_key, - special_outputs_map + special_outputs_map, ) + # DEBUG: Log selected special outputs + logger.info(f"🔍 SELECTED_OUTPUTS: {special_outputs_for_component}") + # Resolve FunctionReference if needed from openhcs.core.pipeline.compiler import FunctionReference + if isinstance(executable_func_or_chain, FunctionReference): executable_func_or_chain = executable_func_or_chain.resolve() - elif isinstance(executable_func_or_chain, tuple) and len(executable_func_or_chain) == 2: + elif ( + isinstance(executable_func_or_chain, tuple) + and len(executable_func_or_chain) == 2 + ): func_or_ref, kwargs = executable_func_or_chain if isinstance(func_or_ref, FunctionReference): executable_func_or_chain = (func_or_ref.resolve(), kwargs) if isinstance(executable_func_or_chain, list): processed_stack = _execute_chain_core( - main_data_stack, executable_func_or_chain, context, - special_inputs_for_component, special_outputs_for_component, axis_id, - device_id, input_memory_type_from_plan, step_index, - dict_key_for_funcplan + main_data_stack, + executable_func_or_chain, + context, + special_inputs_for_component, + special_outputs_for_component, + axis_id, + device_id, + input_memory_type_from_plan, + step_index, + dict_key_for_funcplan, ) - elif callable(executable_func_or_chain) or (isinstance(executable_func_or_chain, tuple) and len(executable_func_or_chain) == 2): + elif callable(executable_func_or_chain) or ( + isinstance(executable_func_or_chain, tuple) + and len(executable_func_or_chain) == 2 + ): # Handle both direct callable and (callable, kwargs) tuple if isinstance(executable_func_or_chain, tuple): actual_func, _ = executable_func_or_chain @@ -692,10 +852,9 @@ def _process_single_pattern_group( # For single functions, apply funcplan filtering like in chain execution funcplan = context.step_plans[step_index].get("funcplan", {}) - func_name = getattr(actual_func, '__name__', 'unknown') + func_name = getattr(actual_func, "__name__", "unknown") execution_key = f"{func_name}_{dict_key_for_funcplan}_0" # Position 0 for single functions - if execution_key in funcplan: outputs_to_save = funcplan[execution_key] filtered_special_outputs_map = _filter_special_outputs_for_function( @@ -706,15 +865,24 @@ def _process_single_pattern_group( filtered_special_outputs_map = {} processed_stack = _execute_function_core( - executable_func_or_chain, main_data_stack, final_base_kwargs, context, - special_inputs_for_component, filtered_special_outputs_map, axis_id, input_memory_type_from_plan, device_id + executable_func_or_chain, + main_data_stack, + final_base_kwargs, + context, + special_inputs_for_component, + filtered_special_outputs_map, + axis_id, + input_memory_type_from_plan, + device_id, ) else: - raise TypeError(f"Invalid executable_func_or_chain: {type(executable_func_or_chain)}") + raise TypeError( + f"Invalid executable_func_or_chain: {type(executable_func_or_chain)}" + ) # 🔍 DEBUG: Check what shape the function actually returned - input_shape = getattr(main_data_stack, 'shape', 'unknown') - output_shape = getattr(processed_stack, 'shape', 'unknown') + input_shape = getattr(main_data_stack, "shape", "unknown") + output_shape = getattr(processed_stack, "shape", "unknown") processed_type = type(processed_stack).__name__ # 🔍 DEBUG: Additional validation logging @@ -722,25 +890,34 @@ def _process_single_pattern_group( if not _is_3d(processed_stack): logger.error("🔍 VALIDATION ERROR: processed_stack is not 3D") logger.error(f"🔍 VALIDATION ERROR: Type: {type(processed_stack)}") - logger.error(f"🔍 VALIDATION ERROR: Shape: {getattr(processed_stack, 'shape', 'no shape attr')}") - logger.error(f"🔍 VALIDATION ERROR: Has ndim: {hasattr(processed_stack, 'ndim')}") - if hasattr(processed_stack, 'ndim'): + logger.error( + f"🔍 VALIDATION ERROR: Shape: {getattr(processed_stack, 'shape', 'no shape attr')}" + ) + logger.error( + f"🔍 VALIDATION ERROR: Has ndim: {hasattr(processed_stack, 'ndim')}" + ) + if hasattr(processed_stack, "ndim"): logger.error(f"🔍 VALIDATION ERROR: ndim value: {processed_stack.ndim}") - raise ValueError(f"Main processing must result in a 3D array, got {getattr(processed_stack, 'shape', 'unknown')}") + raise ValueError( + f"Main processing must result in a 3D array, got {getattr(processed_stack, 'shape', 'unknown')}" + ) # 🔍 DEBUG: Log unstacking operation - - output_slices = unstack_slices( - array=processed_stack, memory_type=output_memory_type_from_plan, gpu_id=device_id, validate_slices=True + array=processed_stack, + memory_type=output_memory_type_from_plan, + gpu_id=device_id, + validate_slices=True, ) # 🔍 DEBUG: Log unstacked result if output_slices: - unstacked_shapes = [getattr(s, 'shape', 'no shape') for s in output_slices[:3]] # First 3 shapes + unstacked_shapes = [ + getattr(s, "shape", "no shape") for s in output_slices[:3] + ] # First 3 shapes # Log values of first slice - if len(output_slices) > 0 and hasattr(output_slices[0], 'min'): + if len(output_slices) > 0 and hasattr(output_slices[0], "min"): first_slice = output_slices[0] # Handle cases where function returns fewer images than inputs (e.g., z-stack flattening, channel compositing) @@ -749,9 +926,13 @@ def _process_single_pattern_group( num_inputs = len(matching_files) if num_outputs < num_inputs: - logger.debug(f"Function returned {num_outputs} images from {num_inputs} inputs - likely flattening operation") + logger.debug( + f"Function returned {num_outputs} images from {num_inputs} inputs - likely flattening operation" + ) elif num_outputs > num_inputs: - logger.warning(f"Function returned more images ({num_outputs}) than inputs ({num_inputs}) - unexpected") + logger.warning( + f"Function returned more images ({num_outputs}) than inputs ({num_inputs}) - unexpected" + ) # Save the output images using batch operations try: @@ -780,13 +961,20 @@ def _process_single_pattern_group( output_paths_batch.append(str(output_path)) # Ensure directory exists - context.filemanager.ensure_directory(str(step_output_dir), Backend.MEMORY.value) + context.filemanager.ensure_directory( + str(step_output_dir), Backend.MEMORY.value + ) # Batch save - context.filemanager.save_batch(output_data, output_paths_batch, Backend.MEMORY.value) + context.filemanager.save_batch( + output_data, output_paths_batch, Backend.MEMORY.value + ) except Exception as e: - logger.error(f"Error saving batch of output slices for pattern {pattern_repr}: {e}", exc_info=True) + logger.error( + f"Error saving batch of output slices for pattern {pattern_repr}: {e}", + exc_info=True, + ) # 🔥 CLEANUP: If function returned fewer images than inputs, delete the unused input files # This prevents unused channel files from remaining in memory after compositing @@ -794,32 +982,48 @@ def _process_single_pattern_group( for j in range(num_outputs, num_inputs): unused_input_filename = matching_files[j] unused_input_path = Path(step_input_dir) / unused_input_filename - if context.filemanager.exists(str(unused_input_path), Backend.MEMORY.value): - context.filemanager.delete(str(unused_input_path), Backend.MEMORY.value) - logger.debug(f"🔥 CLEANUP: Deleted unused input file: {unused_input_filename}") - - + if context.filemanager.exists( + str(unused_input_path), Backend.MEMORY.value + ): + context.filemanager.delete( + str(unused_input_path), Backend.MEMORY.value + ) + logger.debug( + f"🔥 CLEANUP: Deleted unused input file: {unused_input_filename}" + ) - logger.debug(f"Finished pattern group {pattern_repr} in {(time.time() - start_time):.2f}s.") + logger.debug( + f"Finished pattern group {pattern_repr} in {(time.time() - start_time):.2f}s." + ) except Exception as e: import traceback + full_traceback = traceback.format_exc() - logger.error(f"Error processing pattern group {pattern_repr}: {e}", exc_info=True) - logger.error(f"Full traceback for pattern group {pattern_repr}:\n{full_traceback}") + logger.error( + f"Error processing pattern group {pattern_repr}: {e}", exc_info=True + ) + logger.error( + f"Full traceback for pattern group {pattern_repr}:\n{full_traceback}" + ) raise ValueError(f"Failed to process pattern group {pattern_repr}: {e}") from e + class FunctionStep(AbstractStep): # Fields with dedicated editors - hidden from regular ParameterFormManager # but included in ObjectState for tracking and preview - _ui_special_fields = ('func',) + _ui_special_fields = ("func",) def __init__( self, - func: Union[Callable, Tuple[Callable, Dict], List[Union[Callable, Tuple[Callable, Dict]]]]= [], - **kwargs + func: Union[ + Callable, + Tuple[Callable, Dict], + List[Union[Callable, Tuple[Callable, Dict]]], + ] = [], + **kwargs, ): # Generate default name from function if not provided - if 'name' not in kwargs or kwargs['name'] is None: + if "name" not in kwargs or kwargs["name"] is None: actual_func_for_name = func if isinstance(func, tuple): actual_func_for_name = func[0] @@ -829,39 +1033,43 @@ def __init__( actual_func_for_name = first_item[0] elif callable(first_item): actual_func_for_name = first_item - kwargs['name'] = getattr(actual_func_for_name, '__name__', 'FunctionStep') + kwargs["name"] = getattr(actual_func_for_name, "__name__", "FunctionStep") super().__init__(**kwargs) - self.func = func # This is used by prepare_patterns_and_functions at runtime + self.func = func # This is used by prepare_patterns_and_functions at runtime - def process(self, context: 'ProcessingContext', step_index: int) -> None: + def process(self, context: "ProcessingContext", step_index: int) -> None: # Access step plan by index (step_plans keyed by index, not step_id) step_plan = context.step_plans[step_index] # Get step name for logging - step_name = step_plan['step_name'] + step_name = step_plan["step_name"] try: - axis_id = step_plan['axis_id'] - step_input_dir = Path(step_plan['input_dir']) - step_output_dir = Path(step_plan['output_dir']) - variable_components = step_plan['variable_components'] - group_by = step_plan['group_by'] - func_from_plan = step_plan['func'] + axis_id = step_plan["axis_id"] + step_input_dir = Path(step_plan["input_dir"]) + step_output_dir = Path(step_plan["output_dir"]) + variable_components = step_plan["variable_components"] + group_by = step_plan["group_by"] + func_from_plan = step_plan["func"] # special_inputs/outputs are dicts: {'key': 'vfs_path_value'} - special_inputs = step_plan['special_inputs'] - special_outputs = step_plan['special_outputs'] # Should be OrderedDict if order matters - - read_backend = step_plan['read_backend'] - write_backend = step_plan['write_backend'] - input_mem_type = step_plan['input_memory_type'] - output_mem_type = step_plan['output_memory_type'] + special_inputs = step_plan["special_inputs"] + special_outputs = step_plan[ + "special_outputs" + ] # Should be OrderedDict if order matters + + read_backend = step_plan["read_backend"] + write_backend = step_plan["write_backend"] + input_mem_type = step_plan["input_memory_type"] + output_mem_type = step_plan["output_memory_type"] microscope_handler = context.microscope_handler filemanager = context.filemanager # Create path getter for this well - get_paths_for_axis = create_image_path_getter(axis_id, filemanager, microscope_handler) + get_paths_for_axis = create_image_path_getter( + axis_id, filemanager, microscope_handler + ) # Store path getter in step_plan for streaming access step_plan["get_paths_for_axis"] = get_paths_for_axis @@ -869,48 +1077,67 @@ def process(self, context: 'ProcessingContext', step_index: int) -> None: # Get patterns first for bulk preload # Use dynamic filter parameter based on current multiprocessing axis from openhcs.constants import MULTIPROCESSING_AXIS + axis_name = MULTIPROCESSING_AXIS.value filter_kwargs = {f"{axis_name}_filter": [axis_id]} patterns_by_well = microscope_handler.auto_detect_patterns( - str(step_input_dir), # folder_path - filemanager, # filemanager - read_backend, # backend + str(step_input_dir), # folder_path + filemanager, # filemanager + read_backend, # backend extensions=DEFAULT_IMAGE_EXTENSIONS, # extensions - group_by=group_by, # Pass GroupBy enum directly - variable_components=[vc.value for vc in variable_components] if variable_components else [], # variable_components for placeholder logic - **filter_kwargs # Dynamic filter parameter + group_by=group_by, # Pass GroupBy enum directly + variable_components=[vc.value for vc in variable_components] + if variable_components + else [], # variable_components for placeholder logic + **filter_kwargs, # Dynamic filter parameter ) # Debug: Log discovered patterns if axis_id not in patterns_by_well: - logger.warning(f"🔍 PATTERN DISCOVERY: No patterns found for well {axis_id}!") - + logger.warning( + f"🔍 PATTERN DISCOVERY: No patterns found for well {axis_id}!" + ) # Only access gpu_id if the step requires GPU (has GPU memory types) from openhcs.constants.constants import VALID_GPU_MEMORY_TYPES - requires_gpu = (input_mem_type in VALID_GPU_MEMORY_TYPES or - output_mem_type in VALID_GPU_MEMORY_TYPES) - # Ensure variable_components is never None - use default if missing + requires_gpu = ( + input_mem_type in VALID_GPU_MEMORY_TYPES + or output_mem_type in VALID_GPU_MEMORY_TYPES + ) + + # Ensure variable_components is never None - use default if missing if variable_components is None: variable_components = [VariableComponents.SITE] # Default fallback - logger.warning(f"Step {step_index} ({step_name}) had None variable_components, using default [SITE]") + logger.warning( + f"Step {step_index} ({step_name}) had None variable_components, using default [SITE]" + ) if requires_gpu: - device_id = step_plan['gpu_id'] - logger.debug(f"🔥 DEBUG: Step {step_index} gpu_id from plan: {device_id}, input_mem: {input_mem_type}, output_mem: {output_mem_type}") + device_id = step_plan["gpu_id"] + logger.debug( + f"🔥 DEBUG: Step {step_index} gpu_id from plan: {device_id}, input_mem: {input_mem_type}, output_mem: {output_mem_type}" + ) else: device_id = None # CPU-only step - logger.debug(f"🔥 DEBUG: Step {step_index} is CPU-only, input_mem: {input_mem_type}, output_mem: {output_mem_type}") + logger.debug( + f"🔥 DEBUG: Step {step_index} is CPU-only, input_mem: {input_mem_type}, output_mem: {output_mem_type}" + ) - logger.debug(f"🔥 DEBUG: Step {step_index} read_backend: {read_backend}, write_backend: {write_backend}") + logger.debug( + f"🔥 DEBUG: Step {step_index} read_backend: {read_backend}, write_backend: {write_backend}" + ) if not all([axis_id, step_input_dir, step_output_dir]): raise ValueError(f"Plan missing essential keys for step {step_index}") same_dir = str(step_input_dir) == str(step_output_dir) - logger.info(f"Step {step_index} ({step_name}) I/O: read='{read_backend}', write='{write_backend}'.") - logger.info(f"Step {step_index} ({step_name}) Paths: input_dir='{step_input_dir}', output_dir='{step_output_dir}', same_dir={same_dir}") + logger.info( + f"Step {step_index} ({step_name}) I/O: read='{read_backend}', write='{write_backend}'." + ) + logger.info( + f"Step {step_index} ({step_name}) Paths: input_dir='{step_input_dir}', output_dir='{step_output_dir}', same_dir={same_dir}" + ) # Import psutil for memory logging import psutil @@ -929,30 +1156,54 @@ def process(self, context: 'ProcessingContext', step_index: int) -> None: memory_data = filemanager.load_batch(source_paths, read_backend) # Generate conversion paths (input_dir → conversion_dir) - conversion_paths = _generate_materialized_paths(source_paths, Path(step_input_dir), Path(input_conversion_dir)) + conversion_paths = _generate_materialized_paths( + source_paths, Path(step_input_dir), Path(input_conversion_dir) + ) # Parse actual filenames to determine dimensions # Calculate zarr dimensions from conversion paths (which contain the filenames) - n_channels, n_z, n_fields = _calculate_zarr_dimensions(conversion_paths, context.microscope_handler) + n_channels, n_z, n_fields = _calculate_zarr_dimensions( + conversion_paths, context.microscope_handler + ) # Parse well to get row and column for zarr structure - row, col = context.microscope_handler.parser.extract_component_coordinates(axis_id) + row, col = ( + context.microscope_handler.parser.extract_component_coordinates( + axis_id + ) + ) # Save using existing materialized data infrastructure - _save_materialized_data(filemanager, memory_data, conversion_paths, input_conversion_backend, step_plan, context, axis_id) + _save_materialized_data( + filemanager, + memory_data, + conversion_paths, + input_conversion_backend, + step_plan, + context, + axis_id, + ) - logger.info(f"🔬 Converted {len(conversion_paths)} input files to {input_conversion_dir}") + logger.info( + f"🔬 Converted {len(conversion_paths)} input files to {input_conversion_dir}" + ) # Update metadata after conversion conversion_dir = Path(step_plan["input_conversion_dir"]) - zarr_subdir = conversion_dir.name if step_plan["input_conversion_uses_virtual_workspace"] else None + zarr_subdir = ( + conversion_dir.name + if step_plan["input_conversion_uses_virtual_workspace"] + else None + ) _update_metadata_for_zarr_conversion( conversion_dir.parent, step_plan["input_conversion_original_subdir"], zarr_subdir, - context + context, ) - logger.info(f"🔥 STEP: Starting processing for '{step_name}' well {axis_id} (group_by={group_by.name if group_by else None}, variable_components={[vc.name for vc in variable_components] if variable_components else []})") + logger.info( + f"🔥 STEP: Starting processing for '{step_name}' well {axis_id} (group_by={group_by.name if group_by else None}, variable_components={[vc.name for vc in variable_components] if variable_components else []})" + ) if axis_id not in patterns_by_well: raise ValueError( @@ -965,13 +1216,19 @@ def process(self, context: 'ProcessingContext', step_index: int) -> None: if isinstance(patterns_by_well[axis_id], dict): # Grouped patterns (when group_by is set) for comp_val, pattern_list in patterns_by_well[axis_id].items(): - logger.debug(f"🔥 STEP: Component '{comp_val}' has {len(pattern_list)} patterns: {pattern_list}") + logger.debug( + f"🔥 STEP: Component '{comp_val}' has {len(pattern_list)} patterns: {pattern_list}" + ) else: # Ungrouped patterns (when group_by is None) - logger.debug(f"🔥 STEP: Found {len(patterns_by_well[axis_id])} ungrouped patterns: {patterns_by_well[axis_id]}") + logger.debug( + f"🔥 STEP: Found {len(patterns_by_well[axis_id])} ungrouped patterns: {patterns_by_well[axis_id]}" + ) if func_from_plan is None: - raise ValueError(f"Step plan missing 'func' for step: {step_plan.get('step_name', 'Unknown')} (index: {step_index})") + raise ValueError( + f"Step plan missing 'func' for step: {step_plan.get('step_name', 'Unknown')} (index: {step_index})" + ) # 🔄 SEQUENTIAL PROCESSING: Filter patterns BEFORE grouping by group_by component # This ensures sequential filtering works independently of group_by @@ -980,22 +1237,39 @@ def process(self, context: 'ProcessingContext', step_index: int) -> None: seq_component = seq_config.sequential_components[0].value target_value = context.current_sequential_combination[0] - # Filter patterns by sequential component patterns_by_well[axis_id] = _filter_patterns_by_component( patterns_by_well[axis_id], seq_component, target_value, - microscope_handler + microscope_handler, ) - filtered_count = len(patterns_by_well[axis_id]) if isinstance(patterns_by_well[axis_id], list) else sum(len(v) for v in patterns_by_well[axis_id].values()) + filtered_count = ( + len(patterns_by_well[axis_id]) + if isinstance(patterns_by_well[axis_id], list) + else sum(len(v) for v in patterns_by_well[axis_id].values()) + ) # Now group patterns by group_by component (if set) - grouped_patterns, comp_to_funcs, comp_to_base_args = prepare_patterns_and_functions( - patterns_by_well[axis_id], func_from_plan, component=group_by.value if group_by else None + grouped_patterns, comp_to_funcs, comp_to_base_args = ( + prepare_patterns_and_functions( + patterns_by_well[axis_id], + func_from_plan, + component=group_by.value if group_by else None, + ) ) + total_steps = len(context.step_plans) + total_groups = 0 + for pattern_list in grouped_patterns.values(): + total_groups += len(pattern_list) + completed_groups = 0 + if total_groups == 0: + raise ValueError( + f"No pattern groups found for step {step_index} ({step_name}) in well {axis_id}" + ) + # Sequential filtering now happens BEFORE prepare_patterns_and_functions() above # This ensures it works correctly when sequential_components != group_by @@ -1014,66 +1288,121 @@ def process(self, context: 'ProcessingContext', step_index: int) -> None: for comp_val, pattern_list in grouped_patterns.items(): patterns_to_preload.extend(pattern_list) - logger.info(f"� SEQUENTIAL: Preloading {len(patterns_to_preload)} filtered patterns") + logger.info( + f"� SEQUENTIAL: Preloading {len(patterns_to_preload)} filtered patterns" + ) _bulk_preload_step_images( - step_input_dir, step_output_dir, axis_id, read_backend, - patterns_by_well, filemanager, microscope_handler, step_plan["zarr_config"], + step_input_dir, + step_output_dir, + axis_id, + read_backend, + patterns_by_well, + filemanager, + microscope_handler, + step_plan["zarr_config"], patterns_to_preload=patterns_to_preload, - variable_components=[vc.value for vc in variable_components] if variable_components else [] + variable_components=[vc.value for vc in variable_components] + if variable_components + else [], ) else: # Non-sequential: preload all patterns _bulk_preload_step_images( - step_input_dir, step_output_dir, axis_id, read_backend, - patterns_by_well, filemanager, microscope_handler, step_plan["zarr_config"] + step_input_dir, + step_output_dir, + axis_id, + read_backend, + patterns_by_well, + filemanager, + microscope_handler, + step_plan["zarr_config"], ) mem_after_mb = process.memory_info().rss / 1024 / 1024 - logger.info(f"📊 MEMORY: After preload: {mem_after_mb:.1f} MB RSS (+{mem_after_mb - mem_before_mb:.1f} MB)") + logger.info( + f"📊 MEMORY: After preload: {mem_after_mb:.1f} MB RSS (+{mem_after_mb - mem_before_mb:.1f} MB)" + ) # Process each component value for comp_val, current_pattern_list in grouped_patterns.items(): exec_func_or_chain = comp_to_funcs[comp_val] base_kwargs = comp_to_base_args[comp_val] - # Process all patterns for this component value for pattern_item in current_pattern_list: _process_single_pattern_group( - context, pattern_item, exec_func_or_chain, base_kwargs, - step_input_dir, step_output_dir, axis_id, comp_val, - read_backend, write_backend, input_mem_type, output_mem_type, - device_id, same_dir, - special_inputs, special_outputs, + context, + pattern_item, + exec_func_or_chain, + base_kwargs, + step_input_dir, + step_output_dir, + axis_id, + comp_val, + read_backend, + write_backend, + input_mem_type, + output_mem_type, + device_id, + same_dir, + special_inputs, + special_outputs, step_plan["zarr_config"], - variable_components, step_index + variable_components, + step_index, + ) + completed_groups += 1 + emit( + execution_id=context.execution_id, + plate_id=context.plate_id, + axis_id=axis_id, + step_name=step_name, + phase=ProgressPhase.PATTERN_GROUP, + status=ProgressStatus.RUNNING, + completed=completed_groups, + total=total_groups, + percent=(completed_groups / total_groups) * 100.0, + component=str(comp_val), + pattern=str(pattern_item), + worker_slot=context.worker_slot, + owned_wells=context.owned_wells, ) - logger.info(f"🔥 STEP: Completed processing for '{step_name}' well {axis_id}.") + logger.info( + f"🔥 STEP: Completed processing for '{step_name}' well {axis_id}." + ) # 📄 MATERIALIZATION WRITE: Only if not writing to memory if write_backend != Backend.MEMORY.value: memory_paths = get_paths_for_axis(step_output_dir, Backend.MEMORY.value) memory_data = filemanager.load_batch(memory_paths, Backend.MEMORY.value) # Calculate zarr dimensions (ignored by non-zarr backends) - n_channels, n_z, n_fields = _calculate_zarr_dimensions(memory_paths, context.microscope_handler) - row, col = context.microscope_handler.parser.extract_component_coordinates(axis_id) + n_channels, n_z, n_fields = _calculate_zarr_dimensions( + memory_paths, context.microscope_handler + ) + row, col = ( + context.microscope_handler.parser.extract_component_coordinates( + axis_id + ) + ) filemanager.ensure_directory(step_output_dir, write_backend) # Build save kwargs with parser metadata for all backends save_kwargs = { - 'chunk_name': axis_id, - 'zarr_config': step_plan["zarr_config"], - 'n_channels': n_channels, - 'n_z': n_z, - 'n_fields': n_fields, - 'row': row, - 'col': col, - 'parser_name': context.microscope_handler.parser.__class__.__name__, - 'microscope_type': context.microscope_handler.microscope_type + "chunk_name": axis_id, + "zarr_config": step_plan["zarr_config"], + "n_channels": n_channels, + "n_z": n_z, + "n_fields": n_fields, + "row": row, + "col": col, + "parser_name": context.microscope_handler.parser.__class__.__name__, + "microscope_type": context.microscope_handler.microscope_type, } - filemanager.save_batch(memory_data, memory_paths, write_backend, **save_kwargs) + filemanager.save_batch( + memory_data, memory_paths, write_backend, **save_kwargs + ) # 📄 PER-STEP MATERIALIZATION: Additional materialized output if configured if "materialized_output_dir" in step_plan: @@ -1082,12 +1411,26 @@ def process(self, context: 'ProcessingContext', step_index: int) -> None: memory_paths = get_paths_for_axis(step_output_dir, Backend.MEMORY.value) memory_data = filemanager.load_batch(memory_paths, Backend.MEMORY.value) - materialized_paths = _generate_materialized_paths(memory_paths, step_output_dir, Path(materialized_output_dir)) + materialized_paths = _generate_materialized_paths( + memory_paths, step_output_dir, Path(materialized_output_dir) + ) - filemanager.ensure_directory(materialized_output_dir, materialized_backend) - _save_materialized_data(filemanager, memory_data, materialized_paths, materialized_backend, step_plan, context, axis_id) + filemanager.ensure_directory( + materialized_output_dir, materialized_backend + ) + _save_materialized_data( + filemanager, + memory_data, + materialized_paths, + materialized_backend, + step_plan, + context, + axis_id, + ) - logger.info(f"🔬 Materialized {len(materialized_paths)} files to {materialized_output_dir}") + logger.info( + f"🔬 Materialized {len(materialized_paths)} files to {materialized_output_dir}" + ) # 📄 STREAMING: Execute all configured streaming backends from openhcs.core.config import StreamingConfig @@ -1100,7 +1443,9 @@ def process(self, context: 'ProcessingContext', step_index: int) -> None: for key, config_instance in streaming_configs_found: # Get paths at runtime like materialization does step_output_dir = step_plan["output_dir"] - get_paths_for_axis = step_plan["get_paths_for_axis"] # Get the path getter from step_plan + get_paths_for_axis = step_plan[ + "get_paths_for_axis" + ] # Get the path getter from step_plan # Get memory paths (where data actually is) memory_paths = get_paths_for_axis(step_output_dir, Backend.MEMORY.value) @@ -1109,94 +1454,139 @@ def process(self, context: 'ProcessingContext', step_index: int) -> None: # but load from memory paths (where data actually is) if "materialized_output_dir" in step_plan: materialized_output_dir = step_plan["materialized_output_dir"] - streaming_paths = _generate_materialized_paths(memory_paths, step_output_dir, Path(materialized_output_dir)) + streaming_paths = _generate_materialized_paths( + memory_paths, step_output_dir, Path(materialized_output_dir) + ) else: streaming_paths = memory_paths # Load from memory (where data actually is) - streaming_data = filemanager.load_batch(memory_paths, Backend.MEMORY.value) - kwargs = config_instance.get_streaming_kwargs(context) # Pass context for microscope handler access + streaming_data = filemanager.load_batch( + memory_paths, Backend.MEMORY.value + ) + kwargs = config_instance.get_streaming_kwargs( + context + ) # Pass context for microscope handler access # Add pre-built source value for layer/window naming # During pipeline execution: source = step_name kwargs["source"] = step_name # Execute streaming - use streaming_paths (materialized paths) for metadata extraction - filemanager.save_batch(streaming_data, streaming_paths, config_instance.backend.value, **kwargs) + filemanager.save_batch( + streaming_data, + streaming_paths, + config_instance.backend.value, + **kwargs, + ) # Add small delay between image and ROI streaming to prevent race conditions import time + time.sleep(0.1) - logger.info(f"FunctionStep {step_index} ({step_name}) completed for well {axis_id}.") + logger.info( + f"FunctionStep {step_index} ({step_name}) completed for well {axis_id}." + ) # 📄 OPENHCS METADATA: Create metadata file automatically after step completion # Track which backend was actually used for writing files - actual_write_backend = step_plan['write_backend'] + actual_write_backend = step_plan["write_backend"] # Only create OpenHCS metadata for disk/zarr backends, not OMERO # OMERO has its own metadata system and doesn't use openhcs_metadata.json - if actual_write_backend not in [Backend.OMERO_LOCAL.value, Backend.MEMORY.value]: + if actual_write_backend not in [ + Backend.OMERO_LOCAL.value, + Backend.MEMORY.value, + ]: from openhcs.microscopes.openhcs import OpenHCSMetadataGenerator + metadata_generator = OpenHCSMetadataGenerator(context.filemanager) # Main step output metadata - is_pipeline_output = (actual_write_backend != Backend.MEMORY.value) + is_pipeline_output = actual_write_backend != Backend.MEMORY.value metadata_generator.create_metadata( context, - step_plan['output_dir'], + step_plan["output_dir"], actual_write_backend, is_main=is_pipeline_output, - plate_root=step_plan['output_plate_root'], - sub_dir=step_plan['sub_dir'], - results_dir=step_plan.get('analysis_results_dir') # Pass pre-calculated results directory + plate_root=step_plan["output_plate_root"], + sub_dir=step_plan["sub_dir"], + results_dir=step_plan.get( + "analysis_results_dir" + ), # Pass pre-calculated results directory ) # 📄 MATERIALIZED METADATA: Create metadata for materialized directory if it exists # This must be OUTSIDE the main write_backend check because materializations # can happen even when the main step writes to memory - if 'materialized_output_dir' in step_plan: - materialized_backend = step_plan['materialized_backend'] + if "materialized_output_dir" in step_plan: + materialized_backend = step_plan["materialized_backend"] # Only create metadata if materialized backend is also disk/zarr - if materialized_backend not in [Backend.OMERO_LOCAL.value, Backend.MEMORY.value]: + if materialized_backend not in [ + Backend.OMERO_LOCAL.value, + Backend.MEMORY.value, + ]: from openhcs.microscopes.openhcs import OpenHCSMetadataGenerator + metadata_generator = OpenHCSMetadataGenerator(context.filemanager) metadata_generator.create_metadata( context, - step_plan['materialized_output_dir'], + step_plan["materialized_output_dir"], materialized_backend, is_main=False, - plate_root=step_plan['materialized_plate_root'], - sub_dir=step_plan['materialized_sub_dir'], - results_dir=step_plan.get('materialized_analysis_results_dir') # Pass pre-calculated materialized results directory + plate_root=step_plan["materialized_plate_root"], + sub_dir=step_plan["materialized_sub_dir"], + results_dir=step_plan.get( + "materialized_analysis_results_dir" + ), # Pass pre-calculated materialized results directory ) # SPECIAL DATA MATERIALIZATION - special_outputs = step_plan.get('special_outputs', {}) + special_outputs = step_plan.get("special_outputs", {}) if special_outputs: - logger.info(f"🔬 MATERIALIZATION: Starting materialization for {len(special_outputs)} special outputs") + logger.info( + f"🔬 MATERIALIZATION: Starting materialization for {len(special_outputs)} special outputs" + ) # Special outputs ALWAYS use the main materialization backend (disk/zarr), # not the step's write backend (which may be memory for intermediate steps). # This ensures analysis results are always persisted. # Note: _materialize_special_outputs will replace zarr with disk automatically - from openhcs.core.pipeline.materialization_flag_planner import MaterializationFlagPlanner + from openhcs.core.pipeline.materialization_flag_planner import ( + MaterializationFlagPlanner, + ) + vfs_config = context.get_vfs_config() - materialization_backend = MaterializationFlagPlanner._resolve_materialization_backend(context, vfs_config) - self._materialize_special_outputs(filemanager, step_plan, special_outputs, materialization_backend, context) + materialization_backend = ( + MaterializationFlagPlanner._resolve_materialization_backend( + context, vfs_config + ) + ) + self._materialize_special_outputs( + filemanager, + step_plan, + special_outputs, + materialization_backend, + context, + ) logger.info("🔬 MATERIALIZATION: Completed materialization") except Exception as e: import traceback - full_traceback = traceback.format_exc() - logger.error(f"Error in FunctionStep {step_index} ({step_name}): {e}", exc_info=True) - logger.error(f"Full traceback for FunctionStep {step_index} ({step_name}):\n{full_traceback}") - + full_traceback = traceback.format_exc() + logger.error( + f"Error in FunctionStep {step_index} ({step_name}): {e}", exc_info=True + ) + logger.error( + f"Full traceback for FunctionStep {step_index} ({step_name}):\n{full_traceback}" + ) raise - - def _extract_component_metadata(self, context: 'ProcessingContext', component: 'VariableComponents') -> Optional[Dict[str, str]]: + def _extract_component_metadata( + self, context: "ProcessingContext", component: "VariableComponents" + ) -> Optional[Dict[str, str]]: """ Extract component metadata from context cache safely. @@ -1208,20 +1598,19 @@ def _extract_component_metadata(self, context: 'ProcessingContext', component: ' Dictionary mapping component keys to display names, or None if not available """ try: - if hasattr(context, 'metadata_cache') and context.metadata_cache: + if hasattr(context, "metadata_cache") and context.metadata_cache: return context.metadata_cache.get(component, None) else: - logger.debug(f"No metadata_cache available in context for {component.value}") + logger.debug( + f"No metadata_cache available in context for {component.value}" + ) return None except Exception as e: logger.debug(f"Error extracting {component.value} metadata from cache: {e}") return None def _create_openhcs_metadata_for_materialization( - self, - context: 'ProcessingContext', - output_dir: str, - write_backend: str + self, context: "ProcessingContext", output_dir: str, write_backend: str ) -> None: """ Create OpenHCS metadata file for materialization writes. @@ -1237,7 +1626,9 @@ def _create_openhcs_metadata_for_materialization( logger.debug(f"Skipping metadata creation (backend={write_backend})") return - logger.debug(f"Creating metadata for materialization write: {write_backend} -> {output_dir}") + logger.debug( + f"Creating metadata for materialization write: {write_backend} -> {output_dir}" + ) try: # Extract required information @@ -1245,7 +1636,9 @@ def _create_openhcs_metadata_for_materialization( # Check if we have microscope handler for metadata extraction if not context.microscope_handler: - logger.debug("No microscope_handler in context - skipping OpenHCS metadata creation") + logger.debug( + "No microscope_handler in context - skipping OpenHCS metadata creation" + ) return # Get source microscope information @@ -1253,10 +1646,18 @@ def _create_openhcs_metadata_for_materialization( # Extract metadata from source microscope handler try: - grid_dimensions = context.microscope_handler.metadata_handler.get_grid_dimensions(context.input_dir) - pixel_size = context.microscope_handler.metadata_handler.get_pixel_size(context.input_dir) + grid_dimensions = ( + context.microscope_handler.metadata_handler.get_grid_dimensions( + context.input_dir + ) + ) + pixel_size = context.microscope_handler.metadata_handler.get_pixel_size( + context.input_dir + ) except Exception as e: - logger.debug(f"Could not extract grid_dimensions/pixel_size from source: {e}") + logger.debug( + f"Could not extract grid_dimensions/pixel_size from source: {e}" + ) grid_dimensions = [1, 1] # Default fallback pixel_size = 1.0 # Default fallback @@ -1265,11 +1666,19 @@ def _create_openhcs_metadata_for_materialization( image_files = [] if context.filemanager.exists(str(step_output_dir), write_backend): # List files in output directory - files = context.filemanager.list_files(str(step_output_dir), write_backend) + files = context.filemanager.list_files( + str(step_output_dir), write_backend + ) # Filter for image files (common extensions) and convert to strings - image_extensions = {'.tif', '.tiff', '.png', '.jpg', '.jpeg'} - image_files = [str(f) for f in files if Path(f).suffix.lower() in image_extensions] - logger.debug(f"Found {len(image_files)} image files in {step_output_dir}") + image_extensions = {".tif", ".tiff", ".png", ".jpg", ".jpeg"} + image_files = [ + str(f) + for f in files + if Path(f).suffix.lower() in image_extensions + ] + logger.debug( + f"Found {len(image_files)} image files in {step_output_dir}" + ) except Exception as e: logger.debug(f"Could not list image files in output directory: {e}") image_files = [] @@ -1281,19 +1690,32 @@ def _create_openhcs_metadata_for_materialization( metadata = { "microscope_handler_name": context.microscope_handler.microscope_type, "source_filename_parser_name": source_parser_name, - "grid_dimensions": list(grid_dimensions) if hasattr(grid_dimensions, '__iter__') else [1, 1], + "grid_dimensions": list(grid_dimensions) + if hasattr(grid_dimensions, "__iter__") + else [1, 1], "pixel_size": float(pixel_size) if pixel_size is not None else 1.0, "image_files": image_files, - "channels": self._extract_component_metadata(context, VariableComponents.CHANNEL), - "wells": self._extract_component_metadata(context, VariableComponents.WELL), - "sites": self._extract_component_metadata(context, VariableComponents.SITE), - "z_indexes": self._extract_component_metadata(context, VariableComponents.Z_INDEX), - "timepoints": self._extract_component_metadata(context, VariableComponents.TIMEPOINT), - "available_backends": available_backends + "channels": self._extract_component_metadata( + context, VariableComponents.CHANNEL + ), + "wells": self._extract_component_metadata( + context, VariableComponents.WELL + ), + "sites": self._extract_component_metadata( + context, VariableComponents.SITE + ), + "z_indexes": self._extract_component_metadata( + context, VariableComponents.Z_INDEX + ), + "timepoints": self._extract_component_metadata( + context, VariableComponents.TIMEPOINT + ), + "available_backends": available_backends, } # Save metadata file using disk backend (JSON files always on disk) from openhcs.microscopes.openhcs import OpenHCSMetadataHandler + metadata_path = step_output_dir / OpenHCSMetadataHandler.METADATA_FILENAME # Always ensure we can write to the metadata path (delete if exists) @@ -1301,12 +1723,17 @@ def _create_openhcs_metadata_for_materialization( context.filemanager.delete(str(metadata_path), Backend.DISK.value) # Ensure output directory exists on disk - context.filemanager.ensure_directory(str(step_output_dir), Backend.DISK.value) + context.filemanager.ensure_directory( + str(step_output_dir), Backend.DISK.value + ) # Create JSON content - OpenHCS handler expects JSON format import json + json_content = json.dumps(metadata, indent=2) - context.filemanager.save(json_content, str(metadata_path), Backend.DISK.value) + context.filemanager.save( + json_content, str(metadata_path), Backend.DISK.value + ) logger.debug(f"Created OpenHCS metadata file (disk): {metadata_path}") except Exception as e: @@ -1333,7 +1760,14 @@ def _detect_available_backends(self, output_dir: Path) -> Dict[str, bool]: logger.debug(f"Backend detection result: {backends}") return backends - def _build_analysis_filename(self, output_key: str, step_index: int, step_plan: Dict, dict_key: Optional[str] = None, context=None) -> str: + def _build_analysis_filename( + self, + output_key: str, + step_index: int, + step_plan: Dict, + dict_key: Optional[str] = None, + context=None, + ) -> str: """Build analysis result filename from first image path template. Uses first image filename as template to preserve all metadata components. @@ -1346,7 +1780,9 @@ def _build_analysis_filename(self, output_key: str, step_index: int, step_plan: dict_key: Optional channel/component key for dict pattern functions context: Processing context (for accessing microscope handler) """ - memory_paths = step_plan['get_paths_for_axis'](step_plan['output_dir'], Backend.MEMORY.value) + memory_paths = step_plan["get_paths_for_axis"]( + step_plan["output_dir"], Backend.MEMORY.value + ) if not memory_paths: return f"{step_plan['axis_id']}_{output_key}_step{step_index}.roi.zip" @@ -1361,7 +1797,7 @@ def _build_analysis_filename(self, output_key: str, step_index: int, step_plan: for path in memory_paths: filename = Path(path).name metadata = parser.parse_filename(filename) - if metadata and str(metadata.get('channel')) == str(dict_key): + if metadata and str(metadata.get("channel")) == str(dict_key): filtered_paths.append(path) if filtered_paths: @@ -1371,31 +1807,45 @@ def _build_analysis_filename(self, output_key: str, step_index: int, step_plan: base_filename = Path(memory_paths[0]).stem return f"{base_filename}_{output_key}_step{step_index}.roi.zip" - def _materialize_special_outputs(self, filemanager, step_plan, special_outputs, backend, context): + def _materialize_special_outputs( + self, filemanager, step_plan, special_outputs, backend, context + ): """Materialize special outputs (ROIs, cell counts) to disk and streaming backends.""" # Collect backends: main + streaming from openhcs.core.config import StreamingConfig - + backends = [backend] backend_kwargs = {backend: {}} for config in step_plan.values(): if isinstance(config, StreamingConfig): backends.append(config.backend.value) - backend_kwargs[config.backend.value] = config.get_streaming_kwargs(context) + backend_kwargs[config.backend.value] = config.get_streaming_kwargs( + context + ) # Get analysis directory (pre-calculated by compiler) - has_step_mat = 'materialized_output_dir' in step_plan - analysis_output_dir = Path(step_plan['materialized_analysis_results_dir' if has_step_mat else 'analysis_results_dir']) - images_dir = str(step_plan['materialized_output_dir' if has_step_mat else 'output_dir']) + has_step_mat = "materialized_output_dir" in step_plan + analysis_output_dir = Path( + step_plan[ + "materialized_analysis_results_dir" + if has_step_mat + else "analysis_results_dir" + ] + ) + images_dir = str( + step_plan["materialized_output_dir" if has_step_mat else "output_dir"] + ) # Add images_dir and source to all backend kwargs - step_name = step_plan.get('step_name', 'unknown_step') + step_name = step_plan.get("step_name", "unknown_step") for kwargs in backend_kwargs.values(): - kwargs['images_dir'] = images_dir - kwargs['source'] = step_name # Pre-built source value for layer/window naming + kwargs["images_dir"] = images_dir + kwargs["source"] = ( + step_name # Pre-built source value for layer/window naming + ) - filemanager._materialization_context = {'images_dir': images_dir} + filemanager._materialization_context = {"images_dir": images_dir} # Get dict pattern info def _resolve_materializer_inputs(mat_spec, *, dict_key): @@ -1438,7 +1888,9 @@ def _resolve_materializer_inputs(mat_spec, *, dict_key): get_paths_for_axis = step_plan.get("get_paths_for_axis") if get_paths_for_axis is None: - raise ValueError("Step plan missing get_paths_for_axis (cannot resolve materializer inputs)") + raise ValueError( + "Step plan missing get_paths_for_axis (cannot resolve materializer inputs)" + ) paths = get_paths_for_axis(source_dir, source_backend) @@ -1447,7 +1899,10 @@ def _resolve_materializer_inputs(mat_spec, *, dict_key): group_by_key = input_desc.get("group_by") if group_by_key is None: group_by = step_plan.get("group_by") - if group_by is not None and getattr(group_by, "value", None) is not None: + if ( + group_by is not None + and getattr(group_by, "value", None) is not None + ): group_by_key = str(group_by.value) if group_by_key is None: @@ -1466,7 +1921,9 @@ def _resolve_materializer_inputs(mat_spec, *, dict_key): for p in paths: filename = Path(p).name metadata = parser.parse_filename(filename) - if metadata and str(metadata.get(group_by_key)) == str(dict_key): + if metadata and str(metadata.get(group_by_key)) == str( + dict_key + ): filtered_paths.append(p) paths = filtered_paths @@ -1482,16 +1939,16 @@ def _resolve_materializer_inputs(mat_spec, *, dict_key): # Materialize each special output for output_key, output_info in special_outputs.items(): - mat_spec = output_info.get('materialization_spec') + mat_spec = output_info.get("materialization_spec") if not mat_spec: continue - memory_path = output_info['path'] - step_index = step_plan['pipeline_position'] + memory_path = output_info["path"] + step_index = step_plan["pipeline_position"] # For dict patterns, materialize only the channels that produced this output - channels_to_process = output_info.get('group_keys') or [None] - paths_by_group = output_info.get('paths_by_group') or {} + channels_to_process = output_info.get("group_keys") or [None] + paths_by_group = output_info.get("paths_by_group") or {} for dict_key in channels_to_process: # Build channel-specific memory path if needed @@ -1501,25 +1958,37 @@ def _resolve_materializer_inputs(mat_spec, *, dict_key): elif None in paths_by_group: channel_path = paths_by_group[None] else: - from openhcs.core.pipeline.path_planner import PipelinePathPlanner - channel_path = PipelinePathPlanner.build_dict_pattern_path(memory_path, dict_key) + from openhcs.core.pipeline.path_planner import ( + PipelinePathPlanner, + ) + + channel_path = PipelinePathPlanner.build_dict_pattern_path( + memory_path, dict_key + ) else: channel_path = paths_by_group.get(None, memory_path) if not filemanager.exists(channel_path, Backend.MEMORY.value): - logger.info(f"Skipping special output '{output_key}' for group '{dict_key}' - no data saved at {channel_path}") + logger.info( + f"Skipping special output '{output_key}' for group '{dict_key}' - no data saved at {channel_path}" + ) continue # Load data - filemanager.ensure_directory(Path(channel_path).parent, Backend.MEMORY.value) + filemanager.ensure_directory( + Path(channel_path).parent, Backend.MEMORY.value + ) data = filemanager.load(channel_path, Backend.MEMORY.value) # Build analysis filename and path (pass dict_key for channel-specific naming) - filename = self._build_analysis_filename(output_key, step_index, step_plan, dict_key, context) + filename = self._build_analysis_filename( + output_key, step_index, step_plan, dict_key, context + ) analysis_path = analysis_output_dir / filename # Materialize to all backends from openhcs.processing.materialization import materialize + extra_inputs = _resolve_materializer_inputs(mat_spec, dict_key=dict_key) materialize( mat_spec, @@ -1537,7 +2006,7 @@ def _update_metadata_for_zarr_conversion( plate_root: Path, original_subdir: str, zarr_subdir: str | None, - context: 'ProcessingContext' + context: "ProcessingContext", ) -> None: """Update metadata after zarr conversion. @@ -1558,17 +2027,23 @@ def _update_metadata_for_zarr_conversion( is_main=True, plate_root=str(plate_root), sub_dir=zarr_subdir, - skip_if_complete=True + skip_if_complete=True, ) # Set original subdirectory to main=false metadata_path = get_metadata_path(plate_root) writer = AtomicMetadataWriter() - writer.merge_subdirectory_metadata(metadata_path, {original_subdir: {"main": False}}) - logger.info(f"Ensured complete metadata for {zarr_subdir}, set {original_subdir} main=false") + writer.merge_subdirectory_metadata( + metadata_path, {original_subdir: {"main": False}} + ) + logger.info( + f"Ensured complete metadata for {zarr_subdir}, set {original_subdir} main=false" + ) else: # Shared subdirectory - add zarr to available_backends metadata_path = get_metadata_path(plate_root) writer = AtomicMetadataWriter() - writer.merge_subdirectory_metadata(metadata_path, {original_subdir: {"available_backends": {"zarr": True}}}) + writer.merge_subdirectory_metadata( + metadata_path, {original_subdir: {"available_backends": {"zarr": True}}} + ) logger.info(f"Updated metadata: {original_subdir} now has zarr backend") diff --git a/openhcs/core/streaming_config_factory.py b/openhcs/core/streaming_config_factory.py index 9d6b07db2..10c117edb 100644 --- a/openhcs/core/streaming_config_factory.py +++ b/openhcs/core/streaming_config_factory.py @@ -11,7 +11,11 @@ from abc import ABC if TYPE_CHECKING: - from openhcs.core.config import GlobalPipelineConfig, PipelineConfig + from openhcs.core.config import ( + GlobalPipelineConfig, + PipelineConfig, + StreamingConfig, + ) def create_streaming_config( @@ -22,15 +26,16 @@ def create_streaming_config( visualizer_module: str, visualizer_class_name: str, extra_fields: dict = None, - preview_label: str = None -): + preview_label: str = None, + abbreviation: str = None, +) -> Type["StreamingConfig"]: """ Factory to create streaming config classes with minimal boilerplate. - + Eliminates duplication between streaming configs by auto-generating classes from declarative specifications. Adding a new streaming backend requires only 5-10 lines instead of ~50 lines of boilerplate. - + Args: viewer_name: Viewer identifier ('napari', 'fiji', etc.) port: Default port number @@ -39,10 +44,12 @@ def create_streaming_config( visualizer_module: Module path for visualizer class visualizer_class_name: Name of visualizer class extra_fields: Optional dict of {field_name: (type, default_value)} - + preview_label: Short label for list item previews (e.g., "NAP", "FIJI") + abbreviation: Short abbreviation for config class name in grouped previews (e.g., "nap", "fiji") + Returns: Dynamically created streaming config class - + Example: >>> NapariStreamingConfig = create_streaming_config( ... viewer_name='napari', @@ -59,17 +66,20 @@ def create_streaming_config( # Get the global_pipeline_config decorator from config module # It's created by @auto_create_decorator on GlobalPipelineConfig import openhcs.core.config as config_module - global_pipeline_config = getattr(config_module, 'global_pipeline_config', None) + + global_pipeline_config = getattr(config_module, "global_pipeline_config", None) if global_pipeline_config is None: - raise RuntimeError("global_pipeline_config decorator not found. Import openhcs.core.config first.") - + raise RuntimeError( + "global_pipeline_config decorator not found. Import openhcs.core.config first." + ) + # Build class namespace with methods def _get_streaming_kwargs(self, context): kwargs = { "port": self.port, "host": self.host, "transport_mode": self.transport_mode, - "display_config": self + "display_config": self, } # Add extra fields to kwargs if extra_fields: @@ -79,7 +89,7 @@ def _get_streaming_kwargs(self, context): kwargs["microscope_handler"] = context.microscope_handler kwargs["plate_path"] = context.plate_path return kwargs - + def _create_visualizer(self, filemanager, visualizer_config): # Lazy import to avoid circular dependencies module = __import__(visualizer_module, fromlist=[visualizer_class_name]) @@ -91,35 +101,49 @@ def _create_visualizer(self, filemanager, visualizer_config): persistent=self.persistent, port=self.port, display_config=self, - transport_mode=self.transport_mode + transport_mode=self.transport_mode, ) - + + # Compute registry key early (snake_case class name) + import re + cls_name = f"{viewer_name.title()}StreamingConfig" + registry_key = re.sub(r"(? List[int]: """Get all streaming ports for all registered streaming config types. @@ -202,13 +226,20 @@ def get_all_streaming_ports( field_type = field.type # Handle Optional[StreamingConfig] types - if hasattr(typing, 'get_origin') and typing.get_origin(field_type) is typing.Union: + if ( + hasattr(typing, "get_origin") + and typing.get_origin(field_type) is typing.Union + ): # Extract the non-None type from Optional[T] args = typing.get_args(field_type) field_type = next((arg for arg in args if arg is not type(None)), None) # Check if field_type is a StreamingConfig subclass - if field_type is not None and inspect.isclass(field_type) and issubclass(field_type, StreamingConfig): + if ( + field_type is not None + and inspect.isclass(field_type) + and issubclass(field_type, StreamingConfig) + ): # Get the port from the field value or global config field_value = getattr(config, field.name) @@ -234,4 +265,3 @@ def get_all_streaming_ports( ports.extend([port + i for i in range(num_ports_per_type)]) return ports - diff --git a/openhcs/microscopes/openhcs.py b/openhcs/microscopes/openhcs.py index c1e6e2a54..45d2e61c4 100644 --- a/openhcs/microscopes/openhcs.py +++ b/openhcs/microscopes/openhcs.py @@ -822,8 +822,10 @@ def get_primary_backend(self, plate_path: Union[str, Path], filemanager: 'FileMa # 2. Prefer virtual_workspace if available (for plates with workspace_mapping) if 'virtual_workspace' in available_backends_dict and available_backends_dict['virtual_workspace']: - # Register virtual_workspace backend using centralized helper - self._register_virtual_workspace_backend(self.plate_folder, filemanager) + # PERFORMANCE: Only register if not already registered (avoid reloading mappings) + from openhcs.constants.constants import Backend + if Backend.VIRTUAL_WORKSPACE.value not in filemanager.registry: + self._register_virtual_workspace_backend(self.plate_folder, filemanager) return 'virtual_workspace' # 3. Fall back to first available backend (usually disk) diff --git a/openhcs/processing/backends/analysis/__init__.py b/openhcs/processing/backends/analysis/__init__.py index e69633835..69a6304be 100644 --- a/openhcs/processing/backends/analysis/__init__.py +++ b/openhcs/processing/backends/analysis/__init__.py @@ -25,10 +25,22 @@ except ImportError: pass +# Import simple cell counting +try: + from openhcs.processing.backends.analysis.count_cells_simple import \ + count_cells_simple, ThresholdMethod, Foreground +except ImportError: + pass + __all__ = [ # DXF mask pipeline "dxf_mask_pipeline", # Focus analyzer "FocusAnalyzer", + + # Simple cell counting + "count_cells_simple", + "ThresholdMethod", + "Foreground", ] diff --git a/openhcs/processing/backends/analysis/cell_counting_cpu.py b/openhcs/processing/backends/analysis/cell_counting_cpu.py index 7e89f6d5d..0a547d1e4 100644 --- a/openhcs/processing/backends/analysis/cell_counting_cpu.py +++ b/openhcs/processing/backends/analysis/cell_counting_cpu.py @@ -28,7 +28,12 @@ # OpenHCS imports from openhcs.core.memory import numpy as numpy_func from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import materializer_spec, roi_zip_materializer +from openhcs.processing.materialization import ( + MaterializationSpec, + CsvOptions, + JsonOptions, + ROIOptions, +) from openhcs.constants.constants import Backend @@ -88,133 +93,18 @@ class MultiChannelResult: overlap_positions: List[Tuple[float, float]] -def materialize_cell_counts(data: List[Union[CellCountResult, MultiChannelResult]], path: str, filemanager, backends: Union[str, List[str]], backend_kwargs: dict = None) -> str: - """Materialize cell counting results as analysis-ready CSV and JSON formats. - - Args: - data: List of cell count results (single or multi-channel) - path: Output path for results - filemanager: FileManager instance for I/O operations - backends: Single backend string or list of backends to save to - backend_kwargs: Dict mapping backend names to their kwargs (e.g., {'fiji_stream': {'port': 5560}}) - - Returns: - Path to the saved results file (first backend in list) - """ - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - - if backend_kwargs is None: - backend_kwargs = {} - - logger.info(f"🔬 CELL_COUNT_MATERIALIZE: Called with path={path}, data_length={len(data) if data else 0}, backends={backends}") - - # Determine if this is single-channel or multi-channel data - if not data: - logger.warning("🔬 CELL_COUNT_MATERIALIZE: No data to materialize") - return path - - is_multi_channel = isinstance(data[0], MultiChannelResult) - logger.info(f"🔬 CELL_COUNT_MATERIALIZE: is_multi_channel={is_multi_channel}") - - if is_multi_channel: - return _materialize_multi_channel_results(data, path, filemanager, backends, backend_kwargs) - else: - return _materialize_single_channel_results(data, path, filemanager, backends, backend_kwargs) - - -def materialize_segmentation_masks(data: List[np.ndarray], path: str, filemanager, backends: Union[str, List[str]], backend_kwargs: dict = None) -> str: - """Materialize segmentation masks as ROIs and summary. - - Extracts ROIs from labeled masks and saves them via backend-specific format: - - OMERO: Creates omero.model.RoiI objects linked to images - - Napari: Streams shapes layer via ZMQ - - Fiji: Streams ImageJ ROIs via ZMQ - - Disk: Saves JSON file with ROI data - - Args: - data: List of labeled mask arrays (one per z-plane) - path: Output path for ROI data - filemanager: FileManager instance for I/O operations - backends: Single backend string or list of backends to save to - backend_kwargs: Dict mapping backend names to their kwargs (e.g., {'fiji_stream': {'port': 5560}}) - - Returns: - Path to the saved ROI file (first backend in list) - """ - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - - if backend_kwargs is None: - backend_kwargs = {} - - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Called with path={path}, masks_count={len(data) if data else 0}, backends={backends}") - - if not data: - logger.info("🔬 SEGMENTATION_MATERIALIZE: No segmentation masks to materialize (return_segmentation_mask=False)") - # Create empty summary file to indicate no masks were generated - summary_path = path.replace('.pkl', '_segmentation_summary.txt') - summary_content = "No segmentation masks generated (return_segmentation_mask=False)\n" - # Save to all backends - for backend in backends: - filemanager.save(summary_content, summary_path, backend) - return summary_path - - # Extract ROIs from labeled masks (once for all backends) - from polystore.roi import extract_rois_from_labeled_mask - - all_rois = [] - total_cells = 0 - for z_idx, mask in enumerate(data): - rois = extract_rois_from_labeled_mask( - mask, - min_area=10, # Skip tiny regions - extract_contours=True # Extract polygon contours - ) - all_rois.extend(rois) - total_cells += len(rois) - logger.debug(f"🔬 SEGMENTATION_MATERIALIZE: Extracted {len(rois)} ROIs from z-plane {z_idx}") - - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Extracted {total_cells} total ROIs from {len(data)} z-planes") - - # Save ROIs to all backends (streaming backends convert to shapes/ROI objects) - base_path = path.replace('.pkl', '').replace('.roi.zip', '') - roi_path = f"{base_path}_rois.roi.zip" # Extension determines format - - if all_rois: - for backend in backends: - # Get kwargs for this backend (empty dict if not specified) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(all_rois, roi_path, backend, **kwargs) - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Saved {len(all_rois)} ROIs to {backend} backend") - else: - logger.warning(f"🔬 SEGMENTATION_MATERIALIZE: No ROIs extracted (all regions below min_area threshold)") - - # Save summary to all backends (backends will ignore if they don't support text data) - summary_path = f"{base_path}_segmentation_summary.txt" - summary_content = f"Segmentation ROIs: {len(all_rois)} cells\n" - summary_content += f"Z-planes: {len(data)}\n" - if all_rois: - summary_content += f"ROI file: {roi_path}\n" - else: - summary_content += f"No ROIs extracted (all regions below min_area threshold)\n" - - for backend in backends: - # Get kwargs for this backend (empty dict if not specified) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(summary_content, summary_path, backend, **kwargs) - - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Completed, saved to {len(backends)} backends") - - return summary_path - @numpy_func @special_outputs( - ("cell_counts", materializer_spec("cell_counts")), - ("segmentation_masks", roi_zip_materializer()) + ( + "cell_counts", + MaterializationSpec( + JsonOptions(filename_suffix=".json", wrap_list=True), + CsvOptions(filename_suffix="_details.csv"), + primary=0, + ), + ), + ("segmentation_masks", MaterializationSpec(ROIOptions())) ) def count_cells_single_channel( image_stack: np.ndarray, @@ -329,7 +219,14 @@ def count_cells_single_channel( @numpy_func -@special_outputs(("multi_channel_counts", materializer_spec("cell_counts"))) +@special_outputs(( + "multi_channel_counts", + MaterializationSpec( + JsonOptions(filename_suffix=".json", wrap_list=True), + CsvOptions(filename_suffix="_details.csv"), + primary=0, + ), +)) def count_cells_multi_channel( image_stack: np.ndarray, chan_1: int, # Index of first channel (positional arg) @@ -515,211 +412,6 @@ def count_cells_multi_channel( return output_stack, multi_results -def _materialize_single_channel_results(data: List[CellCountResult], path: str, filemanager, backends: Union[str, List[str]], backend_kwargs: dict = None) -> str: - """Materialize single-channel cell counting results.""" - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - - if backend_kwargs is None: - backend_kwargs = {} - - # Generate output file paths based on the input path - # Use clean naming: preserve namespaced path structure, don't duplicate special output key - base_path = path.replace('.pkl', '') - json_path = f"{base_path}.json" - csv_path = f"{base_path}_details.csv" - - # Ensure output directory exists for disk backend - from pathlib import Path - from openhcs.constants.constants import Backend - output_dir = Path(json_path).parent - for backend in backends: - if backend == Backend.DISK.value: - filemanager.ensure_directory(str(output_dir), backend) - - summary = { - "analysis_type": "single_channel_cell_counting", - "total_slices": len(data), - "results_per_slice": [] - } - rows = [] - - total_cells = 0 - for result in data: - total_cells += result.cell_count - - # Add to summary - summary["results_per_slice"].append({ - "slice_index": result.slice_index, - "method": result.method, - "cell_count": result.cell_count, - "avg_cell_area": np.mean(result.cell_areas) if result.cell_areas else 0, - "avg_cell_intensity": np.mean(result.cell_intensities) if result.cell_intensities else 0, - "parameters": result.parameters_used - }) - - # Add individual cell data to CSV - for i, (pos, area, intensity, confidence) in enumerate(zip( - result.cell_positions, result.cell_areas, - result.cell_intensities, result.detection_confidence - )): - rows.append({ - 'slice_index': result.slice_index, - 'cell_id': f"slice_{result.slice_index}_cell_{i}", - 'x_position': pos[0], - 'y_position': pos[1], - 'cell_area': area, - 'cell_intensity': intensity, - 'detection_confidence': confidence, - 'detection_method': result.method - }) - - summary["total_cells_all_slices"] = total_cells - summary["average_cells_per_slice"] = total_cells / len(data) if data else 0 - - # Save JSON summary to all backends (backends will ignore if they don't support text data) - json_content = json.dumps(summary, indent=2, default=str) - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) - - # Get backend instance to check capabilities (polymorphic dispatch) - backend_instance = filemanager._get_backend(backend) - - # Only check exists/delete for backends that support filesystem operations - if backend_instance.requires_filesystem_validation: - if filemanager.exists(json_path, backend): - filemanager.delete(json_path, backend) - - filemanager.save(json_content, json_path, backend, **kwargs) - - # Save CSV details to all backends (backends will ignore if they don't support text data) - if rows: - df = pd.DataFrame(rows) - csv_content = df.to_csv(index=False) - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) - - # Get backend instance to check capabilities (polymorphic dispatch) - backend_instance = filemanager._get_backend(backend) - - # Only check exists/delete for backends that support filesystem operations - if backend_instance.requires_filesystem_validation: - if filemanager.exists(csv_path, backend): - filemanager.delete(csv_path, backend) - - filemanager.save(csv_content, csv_path, backend, **kwargs) - - return json_path - - -def _materialize_multi_channel_results(data: List[MultiChannelResult], path: str, filemanager, backends: Union[str, List[str]], backend_kwargs: dict = None) -> str: - """Materialize multi-channel cell counting and colocalization results.""" - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - - if backend_kwargs is None: - backend_kwargs = {} - - # Generate output file paths based on the input path - # Use clean naming: preserve namespaced path structure, don't duplicate special output key - base_path = path.replace('.pkl', '') - json_path = f"{base_path}.json" - csv_path = f"{base_path}_details.csv" - - # Ensure output directory exists for disk backend - # Skip directory creation for OMERO virtual paths (they don't exist on disk) - from pathlib import Path - output_dir = Path(json_path).parent - for backend in backends: - if backend == Backend.DISK.value and not str(output_dir).startswith('/omero/'): - filemanager.ensure_directory(str(output_dir), backend) - - summary = { - "analysis_type": "multi_channel_cell_counting_colocalization", - "total_slices": len(data), - "colocalization_summary": { - "total_chan_1_cells": 0, - "total_chan_2_cells": 0, - "total_colocalized": 0, - "average_colocalization_percentage": 0 - }, - "results_per_slice": [] - } - - # CSV for detailed analysis (csv_path already defined above) - rows = [] - - total_coloc_pct = 0 - for result in data: - summary["colocalization_summary"]["total_chan_1_cells"] += result.chan_1_results.cell_count - summary["colocalization_summary"]["total_chan_2_cells"] += result.chan_2_results.cell_count - summary["colocalization_summary"]["total_colocalized"] += result.colocalized_count - total_coloc_pct += result.colocalization_percentage - - # Add to summary - summary["results_per_slice"].append({ - "slice_index": result.slice_index, - "chan_1_count": result.chan_1_results.cell_count, - "chan_2_count": result.chan_2_results.cell_count, - "colocalized_count": result.colocalized_count, - "colocalization_percentage": result.colocalization_percentage, - "chan_1_only": result.chan_1_only_count, - "chan_2_only": result.chan_2_only_count, - "colocalization_method": result.colocalization_method, - "colocalization_metrics": result.colocalization_metrics - }) - - # Add colocalization details to CSV - for i, pos in enumerate(result.overlap_positions): - rows.append({ - 'slice_index': result.slice_index, - 'colocalization_id': f"slice_{result.slice_index}_coloc_{i}", - 'x_position': pos[0], - 'y_position': pos[1], - 'colocalization_method': result.colocalization_method - }) - - summary["colocalization_summary"]["average_colocalization_percentage"] = ( - total_coloc_pct / len(data) if data else 0 - ) - - # Save JSON summary to all backends (backends will ignore if they don't support text data) - json_content = json.dumps(summary, indent=2, default=str) - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) - - # Get backend instance to check capabilities (polymorphic dispatch) - backend_instance = filemanager._get_backend(backend) - - # Only check exists/delete for backends that support filesystem operations - if backend_instance.requires_filesystem_validation: - if filemanager.exists(json_path, backend): - filemanager.delete(json_path, backend) - - filemanager.save(json_content, json_path, backend, **kwargs) - - # Save CSV details to all backends (backends will ignore if they don't support text data) - if rows: - df = pd.DataFrame(rows) - csv_content = df.to_csv(index=False) - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) - - # Get backend instance to check capabilities (polymorphic dispatch) - backend_instance = filemanager._get_backend(backend) - - # Only check exists/delete for backends that support filesystem operations - if backend_instance.requires_filesystem_validation: - if filemanager.exists(csv_path, backend): - filemanager.delete(csv_path, backend) - - filemanager.save(csv_content, csv_path, backend, **kwargs) - - return json_path - - def _preprocess_image(image: np.ndarray, gaussian_sigma: float, median_disk_size: int) -> np.ndarray: """Apply preprocessing to enhance cell detection.""" # Gaussian blur to reduce noise diff --git a/openhcs/processing/backends/analysis/cell_counting_cupy.py b/openhcs/processing/backends/analysis/cell_counting_cupy.py index 9545ba40c..9c2263a22 100644 --- a/openhcs/processing/backends/analysis/cell_counting_cupy.py +++ b/openhcs/processing/backends/analysis/cell_counting_cupy.py @@ -42,7 +42,11 @@ # OpenHCS imports from openhcs.core.memory import cupy as cupy_func from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import materializer_spec, tiff_stack_materializer +from openhcs.processing.materialization import ( + MaterializationSpec, + CsvOptions, + JsonOptions, +) from openhcs.constants.constants import Backend @@ -102,90 +106,6 @@ class MultiChannelResult: overlap_positions: List[Tuple[float, float]] -def materialize_cell_counts(data: List[Union[CellCountResult, MultiChannelResult]], path: str, filemanager, backend: str) -> str: - """Materialize cell counting results as analysis-ready CSV and JSON formats.""" - - logger.info(f"🔬 CELL_COUNT_MATERIALIZE: Called with path={path}, data_length={len(data) if data else 0}") - - # Convert CuPy binary_mask to NumPy if present - if data: - for result in data: - if result.binary_mask is not None: - result.binary_mask = cp.asnumpy(result.binary_mask) - if isinstance(result, MultiChannelResult): - if result.chan_1_results.binary_mask is not None: - result.chan_1_results.binary_mask = cp.asnumpy(result.chan_1_results.binary_mask) - if result.chan_2_results.binary_mask is not None: - result.chan_2_results.binary_mask = cp.asnumpy(result.chan_2_results.binary_mask) - - # Determine if this is single-channel or multi-channel data - if not data: - logger.warning("🔬 CELL_COUNT_MATERIALIZE: No data to materialize") - return path - - is_multi_channel = isinstance(data[0], MultiChannelResult) - logger.info(f"🔬 CELL_COUNT_MATERIALIZE: is_multi_channel={is_multi_channel}") - - if is_multi_channel: - return _materialize_multi_channel_results(data, path, filemanager, backend) - else: - return _materialize_single_channel_results(data, path, filemanager, backend) - - -def materialize_segmentation_masks(data: List[cp.ndarray], path: str, filemanager, backend: str) -> str: - """ - Materialize segmentation masks as individual TIFF files. - - Single-backend signature - decorator automatically wraps for multi-backend support. - """ - - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Called with path={path}, backend={backend}, masks_count={len(data) if data else 0}") - - # Convert CuPy arrays to NumPy - if data: - data = [cp.asnumpy(mask) for mask in data] - - if not data: - logger.info("🔬 SEGMENTATION_MATERIALIZE: No segmentation masks to materialize (return_segmentation_mask=False)") - # Create empty summary file to indicate no masks were generated - summary_path = path.replace('.pkl', '_segmentation_summary.txt') - summary_content = "No segmentation masks generated (return_segmentation_mask=False)\n" - filemanager.save(summary_content, summary_path, backend) - return summary_path - - # Generate output file paths based on the input path - base_path = path.replace('.pkl', '') - - # Save each mask as a separate TIFF file - for i, mask in enumerate(data): - mask_filename = f"{base_path}_slice_{i:03d}.tif" - - # Labeled masks must preserve dtype to support >255 labels - # Do NOT convert to uint8 - keep as int32/uint16 - # Save using filemanager with provided backend - filemanager.save(mask, mask_filename, backend) - logger.debug(f"🔬 SEGMENTATION_MATERIALIZE: Saved mask {i} to {mask_filename} (backend={backend})") - - # Return summary path - summary_path = f"{base_path}_segmentation_summary.txt" - summary_content = f"Segmentation masks saved: {len(data)} files\n" - summary_content += f"Base filename pattern: {base_path}_slice_XXX.tif\n" - summary_content += f"Mask dtype: {data[0].dtype}\n" - summary_content += f"Mask shape: {data[0].shape}\n" - - filemanager.save(summary_content, summary_path, backend) - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Completed, saved {len(data)} masks to backend={backend}") - - return summary_path - - -@cupy_func -@special_outputs( - ("cell_counts", materializer_spec("cell_counts")), - ("segmentation_masks", tiff_stack_materializer( - summary_suffix="_segmentation_summary.txt" - )) -) def count_cells_single_channel( image_stack: cp.ndarray, # Detection method and parameters @@ -291,7 +211,14 @@ def count_cells_single_channel( @cupy_func -@special_outputs(("multi_channel_counts", materializer_spec("cell_counts"))) +@special_outputs(( + "multi_channel_counts", + MaterializationSpec( + JsonOptions(filename_suffix=".json", wrap_list=True), + CsvOptions(filename_suffix="_details.csv"), + primary=0, + ), +)) def count_cells_multi_channel( image_stack: cp.ndarray, chan_1: int, # Index of first channel (positional arg) @@ -477,80 +404,6 @@ def count_cells_multi_channel( return output_stack, multi_results -def _materialize_single_channel_results(data: List[CellCountResult], path: str, filemanager, backend: str) -> str: - """Materialize single-channel cell counting results.""" - # Generate output file paths based on the input path - # Use clean naming: preserve namespaced path structure, don't duplicate special output key - base_path = path.replace('.pkl', '') - json_path = f"{base_path}.json" - csv_path = f"{base_path}_details.csv" - - # Ensure output directory exists for disk backend - from pathlib import Path - from openhcs.constants.constants import Backend - output_dir = Path(json_path).parent - if backend == Backend.DISK.value: - filemanager.ensure_directory(str(output_dir), backend) - - summary = { - "analysis_type": "single_channel_cell_counting", - "total_slices": len(data), - "results_per_slice": [] - } - rows = [] - - total_cells = 0 - for result in data: - total_cells += result.cell_count - - # Add to summary - summary["results_per_slice"].append({ - "slice_index": result.slice_index, - "method": result.method, - "cell_count": result.cell_count, - "avg_cell_area": np.mean(result.cell_areas) if result.cell_areas else 0, - "avg_cell_intensity": np.mean(result.cell_intensities) if result.cell_intensities else 0, - "parameters": result.parameters_used - }) - - # Add individual cell data to CSV - for i, (pos, area, intensity, confidence) in enumerate(zip( - result.cell_positions, result.cell_areas, - result.cell_intensities, result.detection_confidence - )): - rows.append({ - 'slice_index': result.slice_index, - 'cell_id': f"slice_{result.slice_index}_cell_{i}", - 'x_position': pos[0], - 'y_position': pos[1], - 'cell_area': area, - 'cell_intensity': intensity, - 'detection_confidence': confidence, - 'detection_method': result.method - }) - - summary["total_cells_all_slices"] = total_cells - summary["average_cells_per_slice"] = total_cells / len(data) if data else 0 - - # Save JSON summary (overwrite if exists) - json_content = json.dumps(summary, indent=2, default=str) - # Remove existing file if it exists using filemanager - if filemanager.exists(json_path, backend): - filemanager.delete(json_path, backend) - filemanager.save(json_content, json_path, backend) - - # Save CSV details (overwrite if exists) - if rows: - df = pd.DataFrame(rows) - csv_content = df.to_csv(index=False) - # Remove existing file if it exists using filemanager - if filemanager.exists(csv_path, backend): - filemanager.delete(csv_path, backend) - filemanager.save(csv_content, csv_path, backend) - - return json_path - - def _materialize_multi_channel_results(data: List[MultiChannelResult], path: str, filemanager, backend: str) -> str: """Materialize multi-channel cell counting and colocalization results.""" # Generate output file paths based on the input path diff --git a/openhcs/processing/backends/analysis/cell_counting_pyclesperanto.py b/openhcs/processing/backends/analysis/cell_counting_pyclesperanto.py index de4c004e1..475f2b4b6 100644 --- a/openhcs/processing/backends/analysis/cell_counting_pyclesperanto.py +++ b/openhcs/processing/backends/analysis/cell_counting_pyclesperanto.py @@ -24,7 +24,11 @@ # OpenHCS imports from openhcs.core.memory import pyclesperanto as pyclesperanto_func from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import materializer_spec, tiff_stack_materializer +from openhcs.processing.materialization import ( + MaterializationSpec, + CsvOptions, + JsonOptions, +) from openhcs.constants.constants import Backend @@ -83,90 +87,6 @@ class MultiChannelResult: overlap_positions: List[Tuple[float, float]] -def materialize_cell_counts(data: List[Union[CellCountResult, MultiChannelResult]], path: str, filemanager, backend: str) -> str: - """Materialize cell counting results as analysis-ready CSV and JSON formats.""" - - logger.info(f"🔬 CELL_COUNT_MATERIALIZE: Called with path={path}, data_length={len(data) if data else 0}, backend={backend}") - - # Convert CLE binary_mask to NumPy if present - if data: - for result in data: - if result.binary_mask is not None: - result.binary_mask = np.asarray(result.binary_mask) - if isinstance(result, MultiChannelResult): - if result.chan_1_results.binary_mask is not None: - result.chan_1_results.binary_mask = np.asarray(result.chan_1_results.binary_mask) - if result.chan_2_results.binary_mask is not None: - result.chan_2_results.binary_mask = np.asarray(result.chan_2_results.binary_mask) - - # Determine if this is single-channel or multi-channel data - if not data: - logger.warning("🔬 CELL_COUNT_MATERIALIZE: No data to materialize") - return path - - is_multi_channel = isinstance(data[0], MultiChannelResult) - logger.info(f"🔬 CELL_COUNT_MATERIALIZE: is_multi_channel={is_multi_channel}") - - if is_multi_channel: - return _materialize_multi_channel_results(data, path, filemanager, backend) - else: - return _materialize_single_channel_results(data, path, filemanager, backend) - - -def materialize_segmentation_masks(data: List[np.ndarray], path: str, filemanager, backend: str) -> str: - """ - Materialize segmentation masks as individual TIFF files. - - Single-backend signature - decorator automatically wraps for multi-backend support. - """ - - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Called with path={path}, backend={backend}, masks_count={len(data) if data else 0}") - - # Convert CLE arrays to NumPy - if data: - data = [np.asarray(mask) for mask in data] - - if not data: - logger.info("🔬 SEGMENTATION_MATERIALIZE: No segmentation masks to materialize (return_segmentation_mask=False)") - # Create empty summary file to indicate no masks were generated - summary_path = path.replace('.pkl', '_segmentation_summary.txt') - summary_content = "No segmentation masks generated (return_segmentation_mask=False)\n" - filemanager.save(summary_content, summary_path, backend) - return summary_path - - # Generate output file paths based on the input path - base_path = path.replace('.pkl', '') - - # Save each mask as a separate TIFF file - for i, mask in enumerate(data): - mask_filename = f"{base_path}_slice_{i:03d}.tif" - - # Labeled masks must preserve dtype to support >255 labels - # Do NOT convert to uint8 - keep as int32/uint16 - # Save using filemanager with provided backend - filemanager.save(mask, mask_filename, backend) - logger.debug(f"🔬 SEGMENTATION_MATERIALIZE: Saved mask {i} to {mask_filename} (backend={backend})") - - # Return summary path - summary_path = f"{base_path}_segmentation_summary.txt" - summary_content = f"Segmentation masks saved: {len(data)} files\n" - summary_content += f"Base filename pattern: {base_path}_slice_XXX.tif\n" - summary_content += f"Mask dtype: {data[0].dtype}\n" - summary_content += f"Mask shape: {data[0].shape}\n" - - filemanager.save(summary_content, summary_path, backend) - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Completed, saved {len(data)} masks to backend={backend}") - - return summary_path - - -@pyclesperanto_func -@special_outputs( - ("cell_counts", materializer_spec("cell_counts")), - ("segmentation_masks", tiff_stack_materializer( - summary_suffix="_segmentation_summary.txt" - )) -) def count_cells_single_channel( image_stack: np.ndarray, # Detection method and parameters @@ -273,7 +193,14 @@ def count_cells_single_channel( @pyclesperanto_func -@special_outputs(("multi_channel_counts", materializer_spec("cell_counts"))) +@special_outputs(( + "multi_channel_counts", + MaterializationSpec( + JsonOptions(filename_suffix=".json", wrap_list=True), + CsvOptions(filename_suffix="_details.csv"), + primary=0, + ), +)) def count_cells_multi_channel( image_stack: np.ndarray, chan_1: int, # Index of first channel (positional arg) @@ -459,79 +386,6 @@ def count_cells_multi_channel( return output_stack, multi_results -def _materialize_single_channel_results(data: List[CellCountResult], path: str, filemanager, backend: str) -> str: - """Materialize single-channel cell counting results.""" - # Generate output file paths based on the input path - # Use clean naming: preserve namespaced path structure, don't duplicate special output key - base_path = path.replace('.pkl', '') - json_path = f"{base_path}.json" - csv_path = f"{base_path}_details.csv" - - # Ensure output directory exists for disk backend - from pathlib import Path - output_dir = Path(json_path).parent - if backend == Backend.DISK.value: - filemanager.ensure_directory(str(output_dir), backend) - - summary = { - "analysis_type": "single_channel_cell_counting", - "total_slices": len(data), - "results_per_slice": [] - } - rows = [] - - total_cells = 0 - for result in data: - total_cells += result.cell_count - - # Add to summary - summary["results_per_slice"].append({ - "slice_index": result.slice_index, - "method": result.method, - "cell_count": result.cell_count, - "avg_cell_area": np.mean(result.cell_areas) if result.cell_areas else 0, - "avg_cell_intensity": np.mean(result.cell_intensities) if result.cell_intensities else 0, - "parameters": result.parameters_used - }) - - # Add individual cell data to CSV - for i, (pos, area, intensity, confidence) in enumerate(zip( - result.cell_positions, result.cell_areas, - result.cell_intensities, result.detection_confidence - )): - rows.append({ - 'slice_index': result.slice_index, - 'cell_id': f"slice_{result.slice_index}_cell_{i}", - 'x_position': pos[0], - 'y_position': pos[1], - 'cell_area': area, - 'cell_intensity': intensity, - 'detection_confidence': confidence, - 'detection_method': result.method - }) - - summary["total_cells_all_slices"] = total_cells - summary["average_cells_per_slice"] = total_cells / len(data) if data else 0 - - # Save JSON summary (overwrite if exists) - json_content = json.dumps(summary, indent=2, default=str) - # Remove existing file if it exists using filemanager - if filemanager.exists(json_path, backend): - filemanager.delete(json_path, backend) - filemanager.save(json_content, json_path, backend) - - # Save CSV details (overwrite if exists) - if rows: - df = pd.DataFrame(rows) - csv_content = df.to_csv(index=False) - # Remove existing file if it exists using filemanager - if filemanager.exists(csv_path, backend): - filemanager.delete(csv_path, backend) - filemanager.save(csv_content, csv_path, backend) - - return json_path - - def _materialize_multi_channel_results(data: List[MultiChannelResult], path: str, filemanager, backend: str) -> str: """Materialize multi-channel cell counting and colocalization results.""" # Generate output file paths based on the input path diff --git a/openhcs/processing/backends/analysis/cell_counting_pyclesperanto_simple.py b/openhcs/processing/backends/analysis/cell_counting_pyclesperanto_simple.py index c8b2047df..020b0bcce 100644 --- a/openhcs/processing/backends/analysis/cell_counting_pyclesperanto_simple.py +++ b/openhcs/processing/backends/analysis/cell_counting_pyclesperanto_simple.py @@ -22,7 +22,11 @@ # OpenHCS imports from openhcs.core.memory import pyclesperanto as pyclesperanto_func from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import materializer_spec, tiff_stack_materializer +from openhcs.processing.materialization import ( + MaterializationSpec, + CsvOptions, + JsonOptions, +) from openhcs.constants.constants import Backend @@ -46,139 +50,6 @@ class CellCountResult: binary_mask: np.ndarray = None # Labeled mask for ROI extraction -def materialize_cell_counts(data: List[CellCountResult], path: str, filemanager, backend: str) -> str: - """Materialize cell counting results - compatible with existing system.""" - logger.info(f"🔬 CELL_COUNT_MATERIALIZE: Called with path={path}, data_length={len(data) if data else 0}, backend={backend}") - - if not data: - logger.warning("🔬 CELL_COUNT_MATERIALIZE: No data to materialize") - return path - - # Generate output file paths - base_path = path.replace('.pkl', '') - json_path = f"{base_path}.json" - csv_path = f"{base_path}_details.csv" - - # Ensure output directory exists for disk backend - from pathlib import Path - output_dir = Path(json_path).parent - if backend == Backend.DISK.value: - filemanager.ensure_directory(str(output_dir), backend) - - summary = { - "analysis_type": "single_channel_cell_counting", - "total_slices": len(data), - "results_per_slice": [] - } - rows = [] - - total_cells = 0 - for result in data: - total_cells += result.cell_count - - # Add to summary - summary["results_per_slice"].append({ - "slice_index": result.slice_index, - "method": result.method, - "cell_count": result.cell_count, - "avg_cell_area": np.mean(result.cell_areas) if result.cell_areas else 0, - "avg_cell_intensity": np.mean(result.cell_intensities) if result.cell_intensities else 0, - "parameters": result.parameters_used - }) - - # Add individual cell data to CSV - for i, (pos, area, intensity, confidence) in enumerate(zip( - result.cell_positions, result.cell_areas, - result.cell_intensities, result.detection_confidence - )): - rows.append({ - 'slice_index': result.slice_index, - 'cell_id': f"slice_{result.slice_index}_cell_{i}", - 'x_position': pos[0], - 'y_position': pos[1], - 'cell_area': area, - 'cell_intensity': intensity, - 'detection_confidence': confidence, - 'detection_method': result.method - }) - - summary["total_cells_all_slices"] = total_cells - summary["average_cells_per_slice"] = total_cells / len(data) if data else 0 - - # Save JSON summary - json_content = json.dumps(summary, indent=2, default=str) - if filemanager.exists(json_path, backend): - filemanager.delete(json_path, backend) - filemanager.save(json_content, json_path, backend) - - # Save CSV details - if rows: - df = pd.DataFrame(rows) - csv_content = df.to_csv(index=False) - if filemanager.exists(csv_path, backend): - filemanager.delete(csv_path, backend) - filemanager.save(csv_content, csv_path, backend) - - return json_path - - -def materialize_segmentation_masks(data: List[np.ndarray], path: str, filemanager, backend: str) -> str: - """Materialize segmentation masks as ROIs - compatible with existing system.""" - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Called with path={path}, backend={backend}, masks_count={len(data) if data else 0}") - - if not data: - logger.info("🔬 SEGMENTATION_MATERIALIZE: No segmentation masks to materialize") - summary_path = path.replace('.pkl', '_segmentation_summary.txt') - summary_content = "No segmentation masks generated (return_segmentation_mask=False)\n" - filemanager.save(summary_content, summary_path, backend) - return summary_path - - # Extract ROIs from labeled masks - from polystore.roi import extract_rois_from_labeled_mask - - all_rois = [] - total_cells = 0 - for z_idx, mask in enumerate(data): - rois = extract_rois_from_labeled_mask( - mask, - min_area=10, - extract_contours=True - ) - all_rois.extend(rois) - total_cells += len(rois) - logger.debug(f"🔬 SEGMENTATION_MATERIALIZE: Extracted {len(rois)} ROIs from z-plane {z_idx}") - - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Extracted {total_cells} total ROIs from {len(data)} z-planes") - - # Save ROIs - base_path = path.replace('.pkl', '') - roi_path = f"{base_path}_rois.roi.zip" - - if all_rois: - filemanager.save(all_rois, roi_path, backend) - logger.info(f"🔬 SEGMENTATION_MATERIALIZE: Saved {len(all_rois)} ROIs to {backend} backend") - - # Save summary - summary_path = f"{base_path}_segmentation_summary.txt" - summary_content = f"Segmentation ROIs: {len(all_rois)} cells\n" - summary_content += f"Z-planes: {len(data)}\n" - if all_rois: - summary_content += f"ROI file: {roi_path}\n" - else: - summary_content += f"No ROIs extracted (all regions below min_area threshold)\n" - - filemanager.save(summary_content, summary_path, backend) - - return summary_path - - -@pyclesperanto_func -@special_outputs( - ("cell_counts", materializer_spec("cell_counts")), - ("segmentation_masks", tiff_stack_materializer( - summary_suffix="_segmentation_summary.txt" - )) -) def count_cells_single_channel( image_stack: np.ndarray, # Simplified parameters diff --git a/openhcs/processing/backends/analysis/consolidate_analysis_results.py b/openhcs/processing/backends/analysis/consolidate_analysis_results.py index 54120f9b7..340de4fcd 100644 --- a/openhcs/processing/backends/analysis/consolidate_analysis_results.py +++ b/openhcs/processing/backends/analysis/consolidate_analysis_results.py @@ -8,7 +8,7 @@ Usage: # Standalone df = consolidate_analysis_results("/path/to/results") - + # In pipeline FunctionStep(func=consolidate_analysis_results_pipeline, ...) """ @@ -22,13 +22,17 @@ from openhcs.core.memory import numpy as numpy_func from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import register_materializer, materializer_spec -from openhcs.processing.materialization.core import _generate_output_path +from openhcs.processing.materialization import CsvOptions, MaterializationSpec # Import config classes with TYPE_CHECKING to avoid circular imports from typing import TYPE_CHECKING + if TYPE_CHECKING: - from openhcs.core.config import AnalysisConsolidationConfig, PlateMetadataConfig, GlobalPipelineConfig + from openhcs.core.config import ( + AnalysisConsolidationConfig, + PlateMetadataConfig, + GlobalPipelineConfig, + ) from openhcs.microscopes.microscope_interfaces import FilenameParser logger = logging.getLogger(__name__) @@ -60,9 +64,9 @@ def extract_analysis_type(filename: str, well_id: str) -> str: after_well = name_without_ext[well_end_pos:] # Find first underscore (analysis type starts after this) - if '_' in after_well: + if "_" in after_well: # Remove everything up to and including first underscore - analysis_type = after_well[after_well.index('_') + 1:] + analysis_type = after_well[after_well.index("_") + 1 :] else: # No underscore found, use everything after well ID analysis_type = after_well @@ -73,7 +77,9 @@ def extract_analysis_type(filename: str, well_id: str) -> str: return analysis_type -def create_metaxpress_header(summary_df: pd.DataFrame, plate_metadata: Optional[Dict[str, str]] = None) -> List[List[str]]: +def create_metaxpress_header( + summary_df: pd.DataFrame, plate_metadata: Optional[Dict[str, str]] = None +) -> List[List[str]]: """ Create MetaXpress-style header rows with metadata. @@ -83,33 +89,39 @@ def create_metaxpress_header(summary_df: pd.DataFrame, plate_metadata: Optional[ plate_metadata = {} # Extract plate info from results directory or use defaults - barcode = plate_metadata.get('barcode', 'OpenHCS-Plate') - plate_name = plate_metadata.get('plate_name', 'OpenHCS Analysis Results') - plate_id = plate_metadata.get('plate_id', '00000') - description = plate_metadata.get('description', 'Consolidated analysis results from OpenHCS pipeline') - acquisition_user = plate_metadata.get('acquisition_user', 'OpenHCS') - z_step = plate_metadata.get('z_step', '1') + barcode = plate_metadata.get("barcode", "OpenHCS-Plate") + plate_name = plate_metadata.get("plate_name", "OpenHCS Analysis Results") + plate_id = plate_metadata.get("plate_id", "00000") + description = plate_metadata.get( + "description", "Consolidated analysis results from OpenHCS pipeline" + ) + acquisition_user = plate_metadata.get("acquisition_user", "OpenHCS") + z_step = plate_metadata.get("z_step", "1") # Create header rows matching MetaXpress format header_rows = [ - ['Barcode', barcode], - ['Plate Name', plate_name], - ['Plate ID', plate_id], - ['Description', description], - ['Acquisition User', acquisition_user], - ['Z Step', z_step] + ["Barcode", barcode], + ["Plate Name", plate_name], + ["Plate ID", plate_id], + ["Description", description], + ["Acquisition User", acquisition_user], + ["Z Step", z_step], ] # Pad header rows to match the number of columns in the data num_cols = len(summary_df.columns) for row in header_rows: while len(row) < num_cols: - row.append('') + row.append("") return header_rows -def save_with_metaxpress_header(summary_df: pd.DataFrame, output_path: str, plate_metadata: Optional[Dict[str, str]] = None): +def save_with_metaxpress_header( + summary_df: pd.DataFrame, + output_path: str, + plate_metadata: Optional[Dict[str, str]] = None, +): """ Save DataFrame with MetaXpress-style header structure. """ @@ -130,14 +142,17 @@ def save_with_metaxpress_header(summary_df: pd.DataFrame, output_path: str, plat all_rows = header_rows + data_rows # Write to CSV manually to preserve the exact structure - with open(output_path, 'w', newline='') as f: + with open(output_path, "w", newline="") as f: import csv + writer = csv.writer(f) for row in all_rows: writer.writerow(row) -def auto_summarize_column(series: pd.Series, column_name: str, analysis_type: str) -> Dict[str, Any]: +def auto_summarize_column( + series: pd.Series, column_name: str, analysis_type: str +) -> Dict[str, Any]: """ Automatically summarize a pandas series with MetaXpress-style naming. @@ -156,44 +171,68 @@ def auto_summarize_column(series: pd.Series, column_name: str, analysis_type: st return {} # Create clean analysis type name for grouping - clean_analysis = analysis_type.replace('_', ' ').title() + clean_analysis = analysis_type.replace("_", " ").title() # Create meaningful metric names based on column content if pd.api.types.is_numeric_dtype(clean_series): # Numeric data - focus on key metrics like MetaXpress - if 'count' in column_name.lower() or 'total' in column_name.lower(): + if "count" in column_name.lower() or "total" in column_name.lower(): # Count/total metrics - summary[f"Total {column_name.replace('_', ' ').title()} ({clean_analysis})"] = clean_series.sum() - summary[f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})"] = clean_series.mean() - - elif 'area' in column_name.lower(): + summary[ + f"Total {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = clean_series.sum() + summary[ + f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = clean_series.mean() + + elif "area" in column_name.lower(): # Area metrics - summary[f"Total {column_name.replace('_', ' ').title()} ({clean_analysis})"] = clean_series.sum() - summary[f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})"] = clean_series.mean() - - elif 'length' in column_name.lower() or 'distance' in column_name.lower(): + summary[ + f"Total {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = clean_series.sum() + summary[ + f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = clean_series.mean() + + elif "length" in column_name.lower() or "distance" in column_name.lower(): # Length/distance metrics - summary[f"Total {column_name.replace('_', ' ').title()} ({clean_analysis})"] = clean_series.sum() - summary[f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})"] = clean_series.mean() - - elif 'intensity' in column_name.lower(): + summary[ + f"Total {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = clean_series.sum() + summary[ + f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = clean_series.mean() + + elif "intensity" in column_name.lower(): # Intensity metrics - summary[f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})"] = clean_series.mean() + summary[ + f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = clean_series.mean() - elif 'confidence' in column_name.lower(): + elif "confidence" in column_name.lower(): # Confidence metrics - summary[f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})"] = clean_series.mean() + summary[ + f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = clean_series.mean() else: # Generic numeric metrics - summary[f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})"] = clean_series.mean() + summary[ + f"Mean {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = clean_series.mean() - elif clean_series.dtype == bool or set(clean_series.unique()).issubset({0, 1, True, False}): + elif clean_series.dtype == bool or set(clean_series.unique()).issubset( + {0, 1, True, False} + ): # Boolean data true_count = clean_series.sum() total_count = len(clean_series) - summary[f"Count {column_name.replace('_', ' ').title()} ({clean_analysis})"] = true_count - summary[f"% {column_name.replace('_', ' ').title()} ({clean_analysis})"] = (true_count / total_count) * 100 + summary[f"Count {column_name.replace('_', ' ').title()} ({clean_analysis})"] = ( + true_count + ) + summary[f"% {column_name.replace('_', ' ').title()} ({clean_analysis})"] = ( + true_count / total_count + ) * 100 else: # Categorical/string data - only include if meaningful @@ -201,7 +240,9 @@ def auto_summarize_column(series: pd.Series, column_name: str, analysis_type: st if len(unique_values) <= 5: # Only include if not too many categories value_counts = clean_series.value_counts() most_common = value_counts.index[0] if len(value_counts) > 0 else None - summary[f"Primary {column_name.replace('_', ' ').title()} ({clean_analysis})"] = most_common + summary[ + f"Primary {column_name.replace('_', ' ').title()} ({clean_analysis})" + ] = most_common return summary @@ -220,7 +261,7 @@ def summarize_analysis_file(file_path: str, analysis_type: str) -> Dict[str, Any return {} summary = {} - clean_analysis = analysis_type.replace('_', ' ').title() + clean_analysis = analysis_type.replace("_", " ").title() # Add key file-level metrics first summary[f"Number of Objects ({clean_analysis})"] = len(df) @@ -231,12 +272,30 @@ def summarize_analysis_file(file_path: str, analysis_type: str) -> Dict[str, Any for column in df.columns: # Skip common index/ID columns - if column.lower() in ['index', 'unnamed: 0', 'slice_index', 'cell_id', 'match_id', 'skeleton_id']: + if column.lower() in [ + "index", + "unnamed: 0", + "slice_index", + "cell_id", + "match_id", + "skeleton_id", + ]: continue # Prioritize key metrics col_lower = column.lower() - if any(key in col_lower for key in ['area', 'count', 'length', 'distance', 'intensity', 'confidence', 'branch']): + if any( + key in col_lower + for key in [ + "area", + "count", + "length", + "distance", + "intensity", + "confidence", + "branch", + ] + ): priority_columns.append(column) else: other_columns.append(column) @@ -261,9 +320,9 @@ def summarize_analysis_file(file_path: str, analysis_type: str) -> Dict[str, Any def consolidate_analysis_results( results_directory: str, well_ids: List[str], - consolidation_config: 'AnalysisConsolidationConfig', - plate_metadata_config: 'PlateMetadataConfig', - output_path: Optional[str] = None + consolidation_config: "AnalysisConsolidationConfig", + plate_metadata_config: "PlateMetadataConfig", + output_path: Optional[str] = None, ) -> pd.DataFrame: """ Consolidate analysis results into a single summary table using configuration objects. @@ -280,7 +339,9 @@ def consolidate_analysis_results( results_dir = Path(results_directory) if not results_dir.exists(): - raise FileNotFoundError(f"Results directory does not exist: {results_directory}") + raise FileNotFoundError( + f"Results directory does not exist: {results_directory}" + ) logger.info(f"Consolidating analysis results from: {results_directory}") @@ -288,7 +349,9 @@ def consolidate_analysis_results( logger.info(f"DEBUG: consolidation_config type: {type(consolidation_config)}") logger.info(f"DEBUG: well_pattern: {repr(consolidation_config.well_pattern)}") logger.info(f"DEBUG: file_extensions: {repr(consolidation_config.file_extensions)}") - logger.info(f"DEBUG: exclude_patterns: {repr(consolidation_config.exclude_patterns)}") + logger.info( + f"DEBUG: exclude_patterns: {repr(consolidation_config.exclude_patterns)}" + ) # Find all relevant files all_files = [] @@ -297,7 +360,9 @@ def consolidate_analysis_results( files = list(results_dir.glob(pattern)) all_files.extend([str(f) for f in files]) - logger.info(f"Found {len(all_files)} files with extensions {consolidation_config.file_extensions}") + logger.info( + f"Found {len(all_files)} files with extensions {consolidation_config.file_extensions}" + ) # Apply exclude filters if consolidation_config.exclude_patterns: @@ -306,12 +371,15 @@ def consolidate_analysis_results( if isinstance(exclude_patterns, str): # If it's a string representation of a tuple, convert it back import ast + logger.info(f"DEBUG: exclude_patterns is string: {repr(exclude_patterns)}") try: exclude_patterns = ast.literal_eval(exclude_patterns) logger.info(f"DEBUG: Successfully parsed to: {repr(exclude_patterns)}") except Exception as e: - logger.warning(f"Could not parse exclude_patterns string: {exclude_patterns}, error: {e}") + logger.warning( + f"Could not parse exclude_patterns string: {exclude_patterns}, error: {e}" + ) exclude_patterns = [] filtered_files = [] @@ -321,7 +389,7 @@ def consolidate_analysis_results( filtered_files.append(file_path) all_files = filtered_files logger.info(f"After filtering: {len(all_files)} files to process") - + # Group files by well ID and analysis type wells_data = {} analysis_types = set() @@ -339,7 +407,9 @@ def consolidate_analysis_results( break if not well_id: - logger.warning(f"Could not find any well ID from {well_ids} in filename {filename}, skipping") + logger.warning( + f"Could not find any well ID from {well_ids} in filename {filename}, skipping" + ) continue analysis_type = extract_analysis_type(filename, well_id) @@ -349,15 +419,17 @@ def consolidate_analysis_results( wells_data[well_id] = {} wells_data[well_id][analysis_type] = file_path - - logger.info(f"Processing {len(wells_data)} wells with analysis types: {sorted(analysis_types)}") - + + logger.info( + f"Processing {len(wells_data)} wells with analysis types: {sorted(analysis_types)}" + ) + # Process each well and create summary summary_rows = [] for well_id in sorted(wells_data.keys()): # Always use a consistent well ID column name - well_summary = {'Well': well_id} + well_summary = {"Well": well_id} # Process each analysis type for this well for analysis_type in sorted(analysis_types): @@ -378,10 +450,10 @@ def consolidate_analysis_results( other_cols = [] for col in summary_df.columns: - if col == 'Well': + if col == "Well": continue - if '(' in col and ')' in col: - analysis_name = col.split('(')[-1].replace(')', '') + if "(" in col and ")" in col: + analysis_name = col.split("(")[-1].replace(")", "") if analysis_name not in analysis_groups: analysis_groups[analysis_name] = [] analysis_groups[analysis_name].append(col) @@ -389,7 +461,7 @@ def consolidate_analysis_results( other_cols.append(col) # Reorder columns: Well first, then grouped by analysis type - ordered_cols = ['Well'] + ordered_cols = ["Well"] for analysis_name in sorted(analysis_groups.keys()): ordered_cols.extend(sorted(analysis_groups[analysis_name])) ordered_cols.extend(sorted(other_cols)) @@ -397,12 +469,14 @@ def consolidate_analysis_results( summary_df = summary_df[ordered_cols] else: # Original style: sort all columns alphabetically - if 'Well' in summary_df.columns: - other_cols = [col for col in summary_df.columns if col != 'Well'] - summary_df = summary_df[['Well'] + sorted(other_cols)] - - logger.info(f"Created summary table with {len(summary_df)} wells and {len(summary_df.columns)} metrics") - + if "Well" in summary_df.columns: + other_cols = [col for col in summary_df.columns if col != "Well"] + summary_df = summary_df[["Well"] + sorted(other_cols)] + + logger.info( + f"Created summary table with {len(summary_df)} wells and {len(summary_df.columns)} metrics" + ) + # Save to CSV if output path specified if output_path is None: output_path = str(results_dir / consolidation_config.output_filename) @@ -410,12 +484,14 @@ def consolidate_analysis_results( if consolidation_config.metaxpress_style: # Create plate metadata dictionary from config plate_metadata = { - 'barcode': plate_metadata_config.barcode or f"OpenHCS-{results_dir.name}", - 'plate_name': plate_metadata_config.plate_name or results_dir.name, - 'plate_id': plate_metadata_config.plate_id or str(hash(str(results_dir)) % 100000), - 'description': plate_metadata_config.description or f"Consolidated analysis results from OpenHCS pipeline: {len(summary_df)} wells analyzed", - 'acquisition_user': plate_metadata_config.acquisition_user, - 'z_step': plate_metadata_config.z_step + "barcode": plate_metadata_config.barcode or f"OpenHCS-{results_dir.name}", + "plate_name": plate_metadata_config.plate_name or results_dir.name, + "plate_id": plate_metadata_config.plate_id + or str(hash(str(results_dir)) % 100000), + "description": plate_metadata_config.description + or f"Consolidated analysis results from OpenHCS pipeline: {len(summary_df)} wells analyzed", + "acquisition_user": plate_metadata_config.acquisition_user, + "z_step": plate_metadata_config.z_step, } save_with_metaxpress_header(summary_df, output_path, plate_metadata) @@ -427,51 +503,18 @@ def consolidate_analysis_results( return summary_df -@register_materializer("consolidated_results") -def materialize_consolidated_results( - data: pd.DataFrame, - output_path: str, - filemanager, - backends, - backend_kwargs: dict = None, - spec=None, - context=None, - extra_inputs: dict | None = None, -) -> str: - """Materialize consolidated results DataFrame to CSV using OpenHCS FileManager.""" - try: - options = spec.options if spec is not None else {} - filename_suffix = options.get("filename_suffix", ".csv") - strip_roi = options.get("strip_roi_suffix", False) - output_path = _generate_output_path(output_path, filename_suffix, ".csv", strip_roi=strip_roi) - - if isinstance(backends, str): - backends = [backends] - backend_kwargs = backend_kwargs or {} - - csv_content = data.to_csv(index=False) - - for backend in backends: - if filemanager.exists(output_path, backend): - filemanager.delete(output_path, backend) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(csv_content, output_path, backend, **kwargs) - - logger.info(f"Materialized consolidated results to {output_path}") - return output_path - - except Exception as e: - logger.error(f"Failed to materialize consolidated results: {e}") - raise +## Greenfield: materialization is writer-driven (no custom materializers). @numpy_func -@special_outputs(("consolidated_results", materializer_spec("consolidated_results"))) +@special_outputs( + ("consolidated_results", MaterializationSpec(CsvOptions(filename_suffix=".csv"))) +) def consolidate_analysis_results_pipeline( image_stack: np.ndarray, results_directory: str, - consolidation_config: 'AnalysisConsolidationConfig', - plate_metadata_config: 'PlateMetadataConfig' + consolidation_config: "AnalysisConsolidationConfig", + plate_metadata_config: "PlateMetadataConfig", ) -> tuple[np.ndarray, pd.DataFrame]: """ Pipeline-compatible version of consolidate_analysis_results. @@ -483,7 +526,7 @@ def consolidate_analysis_results_pipeline( results_directory=results_directory, consolidation_config=consolidation_config, plate_metadata_config=plate_metadata_config, - output_path=None # Will be handled by materialization + output_path=None, # Will be handled by materialization ) return image_stack, summary_df @@ -494,7 +537,7 @@ def merge_result_type_summaries( output_path: str, plate_names: Optional[List[str]] = None, plate_folder_name: Optional[str] = None, - plate_id: Optional[str] = None + plate_id: Optional[str] = None, ) -> pd.DataFrame: """ Merge multiple MetaXpress-style summaries from different result types within the SAME plate. @@ -528,7 +571,9 @@ def merge_result_type_summaries( try: # Read MetaXpress CSV, skipping the 6-line header df = pd.read_csv(summary_path, skiprows=6) - result_type = plate_names[i] if plate_names and i < len(plate_names) else f"type_{i}" + result_type = ( + plate_names[i] if plate_names and i < len(plate_names) else f"type_{i}" + ) logger.info(f"Loaded {len(df)} rows from {result_type}") if merged_df is None: @@ -536,12 +581,16 @@ def merge_result_type_summaries( else: # Merge on Well - one row per well with all columns combined # Use outer join to keep all wells from all result types - merged_df = merged_df.merge(df, on='Well', how='outer', suffixes=('', '_dup')) + merged_df = merged_df.merge( + df, on="Well", how="outer", suffixes=("", "_dup") + ) # Drop duplicate columns (keep first occurrence) - dup_cols = [col for col in merged_df.columns if col.endswith('_dup')] + dup_cols = [col for col in merged_df.columns if col.endswith("_dup")] if dup_cols: - logger.info(f"Dropping {len(dup_cols)} duplicate columns from merge") + logger.info( + f"Dropping {len(dup_cols)} duplicate columns from merge" + ) merged_df = merged_df.drop(columns=dup_cols) except Exception as e: @@ -552,27 +601,29 @@ def merge_result_type_summaries( logger.error("No valid summaries could be loaded") return pd.DataFrame() - logger.info(f"Merged into {len(merged_df)} unique wells with {len(merged_df.columns)} total columns") + logger.info( + f"Merged into {len(merged_df)} unique wells with {len(merged_df.columns)} total columns" + ) # Create MetaXpress header for merged summary # Use plate folder name and plate ID if provided if plate_folder_name and plate_id: merged_metadata = { - 'barcode': f"OpenHCS-{plate_folder_name}", - 'plate_name': plate_folder_name, - 'plate_id': plate_id, - 'description': f"Merged analysis from {len(summary_paths)} result types: {', '.join(plate_names[:3]) if plate_names else 'unknown'}", - 'acquisition_user': 'OpenHCS', - 'z_step': '1' + "barcode": f"OpenHCS-{plate_folder_name}", + "plate_name": plate_folder_name, + "plate_id": plate_id, + "description": f"Merged analysis from {len(summary_paths)} result types: {', '.join(plate_names[:3]) if plate_names else 'unknown'}", + "acquisition_user": "OpenHCS", + "z_step": "1", } else: merged_metadata = { - 'barcode': f"OpenHCS-Merged-{len(summary_paths)}ResultTypes", - 'plate_name': f"Merged Analysis ({len(summary_paths)} result types)", - 'plate_id': str(hash(str(summary_paths)) % 100000), - 'description': f"Merged analysis from {len(summary_paths)} result types: {', '.join(plate_names[:3]) if plate_names else 'unknown'}", - 'acquisition_user': 'OpenHCS', - 'z_step': '1' + "barcode": f"OpenHCS-Merged-{len(summary_paths)}ResultTypes", + "plate_name": f"Merged Analysis ({len(summary_paths)} result types)", + "plate_id": str(hash(str(summary_paths)) % 100000), + "description": f"Merged analysis from {len(summary_paths)} result types: {', '.join(plate_names[:3]) if plate_names else 'unknown'}", + "acquisition_user": "OpenHCS", + "z_step": "1", } # Save with MetaXpress header @@ -583,9 +634,7 @@ def merge_result_type_summaries( def consolidate_multi_plate_summaries( - summary_paths: List[str], - output_path: str, - plate_names: Optional[List[str]] = None + summary_paths: List[str], output_path: str, plate_names: Optional[List[str]] = None ) -> pd.DataFrame: """ Consolidate multiple MetaXpress-style summaries from DIFFERENT plates into a single table. @@ -626,9 +675,13 @@ def consolidate_multi_plate_summaries( plate_names.append(plate_dir) if len(plate_names) != len(summary_paths): - raise ValueError(f"plate_names length ({len(plate_names)}) must match summary_paths length ({len(summary_paths)})") + raise ValueError( + f"plate_names length ({len(plate_names)}) must match summary_paths length ({len(summary_paths)})" + ) - logger.info(f"Concatenating {len(summary_paths)} plate summaries (different plates)") + logger.info( + f"Concatenating {len(summary_paths)} plate summaries (different plates)" + ) # Read all summaries and CONCAT (stack rows - never merge different plates) combined_dfs = [] @@ -659,7 +712,9 @@ def consolidate_multi_plate_summaries( # CONCAT all DataFrames (stack rows, keep plates separate) result_df = pd.concat(combined_dfs, ignore_index=True) - logger.info(f"Concatenated {len(combined_dfs)} plates into {len(result_df)} total rows") + logger.info( + f"Concatenated {len(combined_dfs)} plates into {len(result_df)} total rows" + ) # Save as simple CSV result_df.to_csv(output_path, index=False) @@ -671,8 +726,9 @@ def consolidate_multi_plate_summaries( def consolidate_results_directories( results_dirs: List[Path], plate_path: Path, - global_config: 'GlobalPipelineConfig', - filename_parser: Optional['FilenameParser'] = None + analysis_consolidation_config: "AnalysisConsolidationConfig", + plate_metadata_config: "PlateMetadataConfig", + filename_parser: "FilenameParser", ) -> tuple[List[str], List[tuple[str, str]]]: """ Consolidate multiple results directories and create global summary. @@ -684,18 +740,16 @@ def consolidate_results_directories( Args: results_dirs: List of results directory paths to consolidate plate_path: Root plate path (used for determining global output location) - global_config: Global pipeline configuration - filename_parser: Optional filename parser from microscope handler (preferred). - If not provided, falls back to regex pattern from config. + analysis_consolidation_config: Analysis consolidation configuration + plate_metadata_config: Plate metadata configuration + filename_parser: Filename parser from microscope handler Returns: Tuple of (successful_dirs, failed_dirs) where: - successful_dirs: List of successfully consolidated directory names - failed_dirs: List of (dir_name, error_message) tuples for failures """ - from openhcs.core.orchestrator.orchestrator import _get_consolidate_analysis_results - - consolidate_fn = _get_consolidate_analysis_results() + consolidate_fn = consolidate_analysis_results successful_dirs = [] failed_dirs = [] summary_paths = [] @@ -704,10 +758,14 @@ def consolidate_results_directories( for results_dir in results_dirs: csv_files = list(results_dir.glob("*.csv")) # Skip MetaXpress summaries and other consolidated files - csv_files = [f for f in csv_files if not any( - pattern in f.name.lower() - for pattern in ['metaxpress', 'summary', 'consolidated', 'global'] - )] + csv_files = [ + f + for f in csv_files + if not any( + pattern in f.name.lower() + for pattern in ["metaxpress", "summary", "consolidated", "global"] + ) + ] if not csv_files: logger.info(f"Skipping {results_dir} - no CSV files found") @@ -718,22 +776,14 @@ def consolidate_results_directories( for csv_file in csv_files: well_id = None - # Try using filename parser first (preferred - handles all microscope formats) - if filename_parser: - parsed = filename_parser.parse_filename(csv_file.name) - if parsed and 'well' in parsed: - well_id = parsed['well'] - else: - logger.error(f"Parser failed to extract well ID from {csv_file.name}: {parsed}") - - # Fall back to regex pattern from config (case-insensitive) - if not well_id: - import re - # Make pattern case-insensitive by using re.IGNORECASE - pattern = global_config.analysis_consolidation_config.well_pattern - match = re.search(pattern, csv_file.name, re.IGNORECASE) - if match: - well_id = match.group(1).upper() # Normalize to uppercase + # Extract well ID using filename parser (required - handles all microscope formats) + parsed = filename_parser.parse_filename(csv_file.name) + if parsed and "well" in parsed: + well_id = parsed["well"] + else: + logger.error( + f"Parser failed to extract well ID from {csv_file.name}: {parsed}" + ) if well_id: well_ids.add(well_id) @@ -743,19 +793,21 @@ def consolidate_results_directories( logger.warning(f"No well IDs found in {results_dir}, skipping") continue - logger.info(f"Consolidating {len(csv_files)} CSV files from {len(well_ids)} wells in {results_dir}") + logger.info( + f"Consolidating {len(csv_files)} CSV files from {len(well_ids)} wells in {results_dir}" + ) try: consolidate_fn( results_directory=str(results_dir), well_ids=well_ids, - consolidation_config=global_config.analysis_consolidation_config, - plate_metadata_config=global_config.plate_metadata_config + consolidation_config=analysis_consolidation_config, + plate_metadata_config=plate_metadata_config, ) successful_dirs.append(results_dir.name) # Track summary path for global consolidation - summary_filename = global_config.analysis_consolidation_config.output_filename + summary_filename = analysis_consolidation_config.output_filename summary_path = results_dir / summary_filename if summary_path.exists(): summary_paths.append(str(summary_path)) @@ -767,15 +819,25 @@ def consolidate_results_directories( # Step 2: Create global summary by merging result types if multiple directories were consolidated if len(summary_paths) > 1: try: - logger.info(f"Creating global summary from {len(summary_paths)} result type summaries") + logger.info( + f"Creating global summary from {len(summary_paths)} result type summaries" + ) # Use plate_path root for global output global_output_dir = plate_path - global_summary_filename = global_config.analysis_consolidation_config.global_summary_filename + global_summary_filename = ( + analysis_consolidation_config.global_summary_filename + ) global_summary_path = global_output_dir / global_summary_filename # Extract result type names from results directory paths - result_type_names = [results_dir.name for results_dir in results_dirs if (results_dir / global_config.analysis_consolidation_config.output_filename).exists()] + result_type_names = [ + results_dir.name + for results_dir in results_dirs + if ( + results_dir / analysis_consolidation_config.output_filename + ).exists() + ] # Get plate folder name and plate ID from first summary plate_folder_name = plate_path.name @@ -783,10 +845,10 @@ def consolidate_results_directories( if summary_paths: try: # Read Plate ID from first summary's MetaXpress header (line 3) - with open(summary_paths[0], 'r') as f: + with open(summary_paths[0], "r") as f: lines = [next(f) for _ in range(3)] plate_id_line = lines[2] # Line 3: "Plate ID,12345,..." - plate_id = plate_id_line.split(',')[1] + plate_id = plate_id_line.split(",")[1] except: pass @@ -796,7 +858,7 @@ def consolidate_results_directories( output_path=str(global_summary_path), plate_names=result_type_names, plate_folder_name=plate_folder_name, - plate_id=plate_id + plate_id=plate_id, ) logger.info(f"✅ Global summary created: {global_summary_path}") diff --git a/openhcs/processing/backends/analysis/consolidate_special_outputs.py b/openhcs/processing/backends/analysis/consolidate_special_outputs.py index 373c13158..9f44524f4 100644 --- a/openhcs/processing/backends/analysis/consolidate_special_outputs.py +++ b/openhcs/processing/backends/analysis/consolidate_special_outputs.py @@ -23,8 +23,7 @@ from openhcs.core.memory import numpy as numpy_func from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import register_materializer, materializer_spec -from openhcs.processing.materialization.core import _generate_output_path +from openhcs.processing.materialization import CsvOptions, JsonOptions, MaterializationSpec, TextOptions from openhcs.constants.constants import Backend logger = logging.getLogger(__name__) @@ -45,137 +44,7 @@ class WellPatternType(Enum): CUSTOM = "custom" -@register_materializer("consolidated_summary") -def materialize_consolidated_summary( - data: Dict[str, Any], - output_path: str, - filemanager, - backends, - backend_kwargs: dict = None, - spec=None, - context=None, - extra_inputs: dict | None = None, -) -> str: - """ - Materialize consolidated summary data to CSV file. - - Args: - data: Dictionary containing consolidated summary data - output_path: Path where CSV should be saved - filemanager: OpenHCS FileManager instance - well_id: Well identifier - - Returns: - Path to saved CSV file - """ - try: - options = spec.options if spec is not None else {} - filename_suffix = options.get("filename_suffix", ".csv") - strip_roi = options.get("strip_roi_suffix", False) - output_path = _generate_output_path(output_path, filename_suffix, ".csv", strip_roi=strip_roi) - - # Convert to DataFrame - if 'summary_table' in data: - df = pd.DataFrame(data['summary_table']) - else: - # Fallback: create DataFrame from raw data - df = pd.DataFrame([data]) - - # Generate CSV content - csv_content = df.to_csv(index=False) - - if isinstance(backends, str): - backends = [backends] - backend_kwargs = backend_kwargs or {} - - for backend in backends: - if filemanager.exists(output_path, backend): - filemanager.delete(output_path, backend) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(csv_content, output_path, backend, **kwargs) - - logger.info(f"Materialized consolidated summary to {output_path}") - return output_path - - except Exception as e: - logger.error(f"Failed to materialize consolidated summary: {e}") - raise - - -@register_materializer("detailed_report") -def materialize_detailed_report( - data: Dict[str, Any], - output_path: str, - filemanager, - backends, - backend_kwargs: dict = None, - spec=None, - context=None, - extra_inputs: dict | None = None, -) -> str: - """ - Materialize detailed analysis report to text file. - - Args: - data: Dictionary containing analysis data - output_path: Path where report should be saved - filemanager: OpenHCS FileManager instance - well_id: Well identifier - - Returns: - Path to saved report file - """ - try: - options = spec.options if spec is not None else {} - filename_suffix = options.get("filename_suffix", ".txt") - strip_roi = options.get("strip_roi_suffix", False) - output_path = _generate_output_path(output_path, filename_suffix, ".txt", strip_roi=strip_roi) - - report_lines = [] - report_lines.append("="*80) - report_lines.append("OPENHCS SPECIAL OUTPUTS CONSOLIDATION REPORT") - report_lines.append("="*80) - - if 'metadata' in data: - metadata = data['metadata'] - report_lines.append(f"Analysis timestamp: {metadata.get('timestamp', 'Unknown')}") - report_lines.append(f"Total wells processed: {metadata.get('total_wells', 0)}") - report_lines.append(f"Output types detected: {metadata.get('output_types', [])}") - report_lines.append("") - - if 'summary_stats' in data: - stats = data['summary_stats'] - report_lines.append("SUMMARY STATISTICS:") - report_lines.append("-" * 40) - for output_type, type_stats in stats.items(): - report_lines.append(f"\n{output_type.upper()}:") - for metric, value in type_stats.items(): - if isinstance(value, float): - report_lines.append(f" {metric}: {value:.3f}") - else: - report_lines.append(f" {metric}: {value}") - - report_lines.append("\n" + "="*80) - - # Save report - report_content = "\n".join(report_lines) - - if isinstance(backends, str): - backends = [backends] - backend_kwargs = backend_kwargs or {} - - for backend in backends: - if filemanager.exists(output_path, backend): - filemanager.delete(output_path, backend) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(report_content, output_path, backend, **kwargs) - - logger.info(f"Materialized detailed report to {output_path}") - return output_path - - except Exception as e: - logger.error(f"Failed to materialize detailed report: {e}") - raise +## Greenfield: materialization is writer-driven (no custom materializers). def extract_well_id(filename: str, pattern: str = WellPatternType.STANDARD_96.value) -> Optional[str]: @@ -273,8 +142,13 @@ def aggregate_series(series: pd.Series, strategy: AggregationStrategy) -> Dict[s @numpy_func @special_outputs( - ("consolidated_summary", materializer_spec("consolidated_summary")), - ("detailed_report", materializer_spec("detailed_report")) + ("consolidated_summary", MaterializationSpec(CsvOptions(filename_suffix=".csv"))), + ( + "detailed_report", + MaterializationSpec( + TextOptions(filename_suffix=".txt") + ), + ), ) def consolidate_special_outputs( image_stack: np.ndarray, diff --git a/openhcs/processing/backends/analysis/count_cells_simple.py b/openhcs/processing/backends/analysis/count_cells_simple.py new file mode 100644 index 000000000..4e0a3332c --- /dev/null +++ b/openhcs/processing/backends/analysis/count_cells_simple.py @@ -0,0 +1,143 @@ +""" +Simple cell counting using thresholding and connected component labeling. + +This module provides a straightforward cell counting function with basic +thresholding methods and size filtering. +""" + +from openhcs.core.memory import numpy +from openhcs.core.pipeline.function_contracts import special_outputs +from openhcs.processing.materialization import MaterializationSpec, CsvOptions, ROIOptions + +from enum import Enum +from typing import Tuple, List + +import numpy as np +from scipy import ndimage as ndi +from skimage.filters import threshold_otsu, threshold_li, threshold_yen + + +class ThresholdMethod(str, Enum): + """Thresholding methods for cell detection.""" + OTSU = "otsu" + LI = "li" + YEN = "yen" + PERCENTILE = "percentile" + MANUAL = "manual" + + +class Foreground(str, Enum): + """Foreground type for thresholding.""" + BRIGHT = "bright" + DARK = "dark" + + +# Make the Enums importable/stable for multiprocessing/ZMQ pickling +import openhcs.processing.backends.analysis.count_cells_simple as _count_cells_simple + +ThresholdMethod.__module__ = _count_cells_simple.__name__ +setattr(_count_cells_simple, "ThresholdMethod", ThresholdMethod) + +Foreground.__module__ = _count_cells_simple.__name__ +setattr(_count_cells_simple, "Foreground", Foreground) + + +@numpy +@special_outputs( + ( + "cell_counts", + MaterializationSpec(CsvOptions(fields=["slice_index", "cell_count"])) + ), + ("segmentation_masks", MaterializationSpec(ROIOptions())) +) +def count_cells_simple( + image, + threshold_method: ThresholdMethod = ThresholdMethod.OTSU, + threshold: float = 0.5, + threshold_percentile: float = 99.0, + foreground: Foreground = Foreground.BRIGHT, + min_size: int = 20, + max_size: int = 100000 +) -> Tuple[np.ndarray, List[dict], List[np.ndarray]]: + """ + Count cells in a 3D image using simple thresholding and connected component labeling. + + Args: + image: Input image as 3D array (C, Y, X) + threshold_method: One of {"otsu", "li", "yen", "percentile", "manual"} + threshold: Manual threshold value (used when threshold_method="manual") + threshold_percentile: Percentile in [0,100] (used when threshold_method="percentile") + foreground: "bright" (cells brighter than background) or "dark" + min_size: Minimum pixel size to keep an object + max_size: Maximum pixel size to keep an object (filters out large artifacts) + + Returns: + Tuple: + - image (unchanged) + - list of dicts with cell count per slice + - list of segmentation masks (one per slice) + """ + results = [] + masks = [] + + for i, slice_data in enumerate(image): + # Convert enum strings to enum objects if needed + if isinstance(threshold_method, str): + threshold_method = ThresholdMethod(threshold_method) + if isinstance(foreground, str): + foreground = Foreground(foreground) + + # Compute threshold in the native intensity scale of slice_data + if threshold_method == ThresholdMethod.OTSU: + thr = threshold_otsu(slice_data) + elif threshold_method == ThresholdMethod.LI: + thr = threshold_li(slice_data) + elif threshold_method == ThresholdMethod.YEN: + thr = threshold_yen(slice_data) + elif threshold_method == ThresholdMethod.PERCENTILE: + thr = float(np.percentile(slice_data, threshold_percentile)) + elif threshold_method == ThresholdMethod.MANUAL: + # If user supplies a fractional threshold in [0,1] but the data is scaled + # (e.g. uint16 or normalized to ~[0,65535]), interpret it as fraction-of-max. + thr = float(threshold) + if 0.0 <= thr <= 1.0: + max_val = float(np.max(slice_data)) + if max_val > 1.0: + thr *= max_val + else: + raise ValueError(f"Unknown threshold_method: {threshold_method!r}") + + # Apply threshold + if foreground == Foreground.BRIGHT: + binary = slice_data > thr + elif foreground == Foreground.DARK: + binary = slice_data < thr + else: + raise ValueError(f"Unknown foreground: {foreground!r} (expected 'bright' or 'dark')") + + # Label connected components + labeled, num_objects = ndi.label(binary) + + # Filter objects by size (min and max) + if num_objects > 0: + sizes = ndi.sum(binary, labeled, index=range(1, num_objects + 1)) + keep_labels = [ + idx + 1 for idx, size in enumerate(sizes) + if min_size <= size <= max_size + ] + + filtered = np.isin(labeled, keep_labels) + labeled_filtered, final_count = ndi.label(filtered) + else: + labeled_filtered = np.zeros_like(labeled) + final_count = 0 + + results.append({ + "slice_index": i, + "cell_count": int(final_count) + }) + + masks.append(labeled_filtered.astype(np.int32, copy=False)) + + return image, results, masks + diff --git a/openhcs/processing/backends/analysis/hmm_axon.py b/openhcs/processing/backends/analysis/hmm_axon.py index 832b61f2e..755b5e18a 100644 --- a/openhcs/processing/backends/analysis/hmm_axon.py +++ b/openhcs/processing/backends/analysis/hmm_axon.py @@ -17,8 +17,13 @@ from skimage.morphology import skeletonize from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import register_materializer, materializer_spec, tiff_stack_materializer -from openhcs.processing.materialization.core import _generate_output_path +from openhcs.processing.materialization import ( + MaterializationSpec, + CsvOptions, + JsonOptions, + TextOptions, + TiffStackOptions, +) from openhcs.core.utils import optional_import from openhcs.constants.constants import Backend @@ -34,92 +39,6 @@ alva_branch = None -@register_materializer("hmm_analysis_rrs") -def materialize_hmm_analysis( - hmm_analysis_data: Dict[str, Any], - path: str, - filemanager, - backends, - backend_kwargs: dict = None, - spec=None, - context=None, - extra_inputs: dict | None = None, -) -> str: - """ - Materialize HMM neurite tracing analysis results to disk. - - Creates multiple output files: - - JSON file with graph data and summary metrics - - GraphML file with the NetworkX graph - - CSV file with edge data - - Args: - hmm_analysis_data: The HMM analysis results dictionary - path: Base path for output files (from special output path) - filemanager: FileManager instance for consistent I/O - backend: Backend to use for materialization - **kwargs: Additional materialization options - - Returns: - str: Path to the primary output file (JSON summary) - """ - import json - import networkx as nx - from pathlib import Path - from openhcs.constants.constants import Backend - - # Generate output file paths - base_path = _generate_output_path(path, "", "", strip_roi=False) - json_path = f"{base_path}.json" - graphml_path = f"{base_path}_graph.graphml" - csv_path = f"{base_path}_edges.csv" - - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - backend_kwargs = backend_kwargs or {} - - # GraphML requires disk backend - for backend in backends: - if backend != Backend.DISK.value: - raise ValueError("hmm_analysis requires disk backend for GraphML output") - - output_dir = Path(json_path).parent - filemanager.ensure_directory(str(output_dir), Backend.DISK.value) - - # 1. Save summary and metadata as JSON (primary output) - summary_data = { - 'analysis_type': 'hmm_neurite_tracing', - 'summary': hmm_analysis_data['summary'], - 'metadata': hmm_analysis_data['metadata'] - } - json_content = json.dumps(summary_data, indent=2, default=str) - filemanager.save(json_content, json_path, Backend.DISK.value) - - # 2. Save NetworkX graph as GraphML - graph = hmm_analysis_data['graph'] - if graph and graph.number_of_nodes() > 0: - # Use direct file I/O for GraphML (NetworkX doesn't support string I/O) - # Note: NetworkX requires actual file path, not compatible with OMERO backend - nx.write_graphml(graph, graphml_path) - - # 3. Save edge data as CSV - if graph.number_of_edges() > 0: - import pandas as pd - edge_data = [] - for u, v, data in graph.edges(data=True): - edge_info = { - 'source_x': u[0], 'source_y': u[1], - 'target_x': v[0], 'target_y': v[1], - **data # Include any edge attributes - } - edge_data.append(edge_info) - - edge_df = pd.DataFrame(edge_data) - csv_content = edge_df.to_csv(index=False) - filemanager.save(csv_content, csv_path, Backend.DISK.value) - - return json_path def materialize_trace_visualizations(data: List[np.ndarray], path: str, filemanager, backend: str) -> str: @@ -424,11 +343,25 @@ def create_visualization_array( raise ValueError(f"Unknown visualization mode: {mode}") @special_outputs( - ("hmm_analysis", materializer_spec("hmm_analysis_rrs", allowed_backends=[Backend.DISK.value])), - ("trace_visualizations", tiff_stack_materializer( - normalize_uint8=True, - summary_suffix="_trace_summary.txt" - )) + ( + "hmm_analysis", + MaterializationSpec( + JsonOptions(source="summary", filename_suffix=".json"), + TextOptions(source="graphml", filename_suffix="_graph.graphml"), + CsvOptions(source="edges", filename_suffix="_edges.csv"), + primary=0, + allowed_backends=[Backend.DISK.value], + ), + ), + ( + "trace_visualizations", + MaterializationSpec( + TiffStackOptions( + normalize_uint8=True, + summary_suffix="_trace_summary.txt", + ) + ), + ), ) @numpy def trace_neurites_rrs_alva( @@ -548,6 +481,7 @@ def _compile_hmm_analysis_results( ) -> Dict[str, Any]: """Compile comprehensive HMM analysis results.""" from datetime import datetime + import io # Compute summary metrics from the graph num_nodes = combined_graph.number_of_nodes() @@ -589,10 +523,31 @@ def _compile_hmm_analysis_results( 'processing_timestamp': datetime.now().isoformat(), } + graphml: str = "" + edges: List[Dict[str, Any]] = [] + if combined_graph and combined_graph.number_of_nodes() > 0: + try: + buf = io.StringIO() + nx.write_graphml(combined_graph, buf) + graphml = buf.getvalue() + except Exception: + graphml = "" + + for u, v, edge_attrs in combined_graph.edges(data=True): + edges.append( + { + "source_x": u[0], + "source_y": u[1], + "target_x": v[0], + "target_y": v[1], + **(edge_attrs or {}), + } + ) + return { - 'summary': summary, - 'graph': combined_graph, - 'metadata': metadata + 'summary': {**summary, **metadata}, + 'graphml': graphml, + 'edges': edges, } diff --git a/openhcs/processing/backends/analysis/hmm_axon_torbi.py b/openhcs/processing/backends/analysis/hmm_axon_torbi.py index 2a22dacfc..09d2537e7 100644 --- a/openhcs/processing/backends/analysis/hmm_axon_torbi.py +++ b/openhcs/processing/backends/analysis/hmm_axon_torbi.py @@ -17,8 +17,13 @@ from skimage.morphology import skeletonize from openhcs.core.memory import torch as torch_func from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import register_materializer, materializer_spec, tiff_stack_materializer -from openhcs.processing.materialization.core import _generate_output_path +from openhcs.processing.materialization import ( + MaterializationSpec, + CsvOptions, + JsonOptions, + TextOptions, + TiffStackOptions, +) from openhcs.constants.constants import Backend # Import torch using the established optional import pattern @@ -29,87 +34,7 @@ torbi = optional_import("torbi") -@register_materializer("hmm_analysis_torbi") -def materialize_hmm_analysis( - hmm_analysis_data: Dict[str, Any], - path: str, - filemanager, - backends, - backend_kwargs: dict = None, - spec=None, - context=None, - extra_inputs: dict | None = None, -) -> str: - """ - Materialize HMM neurite tracing analysis results to disk. - - Creates multiple output files: - - JSON file with graph data and summary metrics - - GraphML file with the NetworkX graph - - CSV file with edge data - - Args: - hmm_analysis_data: The HMM analysis results dictionary - path: Base path for output files (from special output path) - filemanager: FileManager instance for consistent I/O - **kwargs: Additional materialization options - - Returns: - str: Path to the primary output file (JSON summary) - """ - import json - import networkx as nx - from pathlib import Path - # Backends are required to be disk-only for GraphML output - if isinstance(backends, str): - backends = [backends] - backend_kwargs = backend_kwargs or {} - for backend in backends: - if backend != Backend.DISK.value: - raise ValueError("hmm_analysis requires disk backend for GraphML output") - - # Generate output file paths - base_path = _generate_output_path(path, "", "", strip_roi=False) - json_path = f"{base_path}.json" - graphml_path = f"{base_path}_graph.graphml" - csv_path = f"{base_path}_edges.csv" - - # Ensure output directory exists - output_dir = Path(json_path).parent - filemanager.ensure_directory(str(output_dir), Backend.DISK.value) - - # 1. Save summary and metadata as JSON (primary output) - summary_data = { - 'analysis_type': 'hmm_neurite_tracing_torbi', - 'summary': hmm_analysis_data['summary'], - 'metadata': hmm_analysis_data['metadata'] - } - json_content = json.dumps(summary_data, indent=2, default=str) - filemanager.save(json_content, json_path, Backend.DISK.value) - - # 2. Save NetworkX graph as GraphML - graph = hmm_analysis_data['graph'] - if graph and graph.number_of_nodes() > 0: - # Use direct file I/O for GraphML (NetworkX doesn't support string I/O) - nx.write_graphml(graph, graphml_path) - - # 3. Save edge data as CSV - if graph.number_of_edges() > 0: - import pandas as pd - edge_data = [] - for u, v, data in graph.edges(data=True): - edge_info = { - 'source_x': u[0], 'source_y': u[1], - 'target_x': v[0], 'target_y': v[1], - **data # Include any edge attributes - } - edge_data.append(edge_info) - - edge_df = pd.DataFrame(edge_data) - csv_content = edge_df.to_csv(index=False) - filemanager.save(csv_content, csv_path, Backend.DISK.value) - - return json_path +## Greenfield: materialization is writer-driven (no custom materializers). def materialize_trace_visualizations(data: List[np.ndarray], path: str, filemanager) -> str: @@ -483,10 +408,21 @@ def create_visualization_array( raise ValueError(f"Unknown visualization mode: {mode}") @special_outputs( - ("hmm_analysis", materializer_spec("hmm_analysis_torbi", allowed_backends=[Backend.DISK.value])), - ("trace_visualizations", tiff_stack_materializer( - normalize_uint8=True, - summary_suffix="_trace_summary.txt" + ( + "hmm_analysis", + MaterializationSpec( + JsonOptions(source="summary", filename_suffix=".json"), + TextOptions(source="graphml", filename_suffix="_graph.graphml"), + CsvOptions(source="edges", filename_suffix="_edges.csv"), + primary=0, + allowed_backends=[Backend.DISK.value], + ), + ), + ("trace_visualizations", MaterializationSpec( + TiffStackOptions( + normalize_uint8=True, + summary_suffix="_trace_summary.txt" + ) )) ) @torch_func @@ -611,6 +547,7 @@ def _compile_hmm_analysis_results( ) -> Dict[str, Any]: """Compile comprehensive HMM analysis results for torbi version.""" from datetime import datetime + import io # Compute summary metrics from the graph num_nodes = combined_graph.number_of_nodes() @@ -653,10 +590,31 @@ def _compile_hmm_analysis_results( 'gpu_accelerated': True, } + graphml: str = "" + edges: List[Dict[str, Any]] = [] + if combined_graph and combined_graph.number_of_nodes() > 0: + try: + buf = io.StringIO() + nx.write_graphml(combined_graph, buf) + graphml = buf.getvalue() + except Exception: + graphml = "" + + for u, v, edge_attrs in combined_graph.edges(data=True): + edges.append( + { + "source_x": u[0], + "source_y": u[1], + "target_x": v[0], + "target_y": v[1], + **(edge_attrs or {}), + } + ) + return { - 'summary': summary, - 'graph': combined_graph, - 'metadata': metadata + 'summary': {**summary, **metadata}, + 'graphml': graphml, + 'edges': edges, } diff --git a/openhcs/processing/backends/analysis/multi_template_matching.py b/openhcs/processing/backends/analysis/multi_template_matching.py index 6bee2d0f6..7040834a0 100644 --- a/openhcs/processing/backends/analysis/multi_template_matching.py +++ b/openhcs/processing/backends/analysis/multi_template_matching.py @@ -5,6 +5,8 @@ to detect and crop regions of interest in image stacks. """ +from __future__ import annotations + import numpy as np import cv2 from typing import Tuple, List, Dict, Any, Optional, Union @@ -22,8 +24,48 @@ from openhcs.core.memory import numpy as numpy_func from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import register_materializer, materializer_spec -from openhcs.processing.materialization.core import _generate_output_path +from openhcs.processing.materialization import CsvOptions, MaterializationSpec + + +def _mtm_row_unpacker(result: TemplateMatchResult) -> List[Dict[str, Any]]: + """Expand one TemplateMatchResult into multiple CSV rows.""" + rows: List[Dict[str, Any]] = [] + slice_idx = result.slice_index + + for i, match in enumerate(result.matches or []): + # MTM hits format: [label, bbox, score] where bbox is (x, y, width, height) + if len(match) >= 3: + template_label, bbox, score = match[0], match[1], match[2] + x, y, w, h = bbox if len(bbox) >= 4 else (0, 0, 0, 0) + rows.append( + { + "match_id": f"slice_{slice_idx}_match_{i}", + "bbox_x": x, + "bbox_y": y, + "bbox_width": w, + "bbox_height": h, + "confidence_score": score, + "template_name": template_label, + "is_best_match": (match == result.best_match), + "was_cropped": result.crop_bbox is not None, + } + ) + else: + rows.append( + { + "match_id": f"slice_{slice_idx}_match_{i}", + "bbox_x": 0, + "bbox_y": 0, + "bbox_width": 0, + "bbox_height": 0, + "confidence_score": 0.0, + "template_name": "malformed_match", + "is_best_match": False, + "was_cropped": result.crop_bbox is not None, + } + ) + + return rows @dataclass class TemplateMatchResult: @@ -37,126 +79,13 @@ class TemplateMatchResult: best_rotation_angle: float # Angle of best matching template error_message: Optional[str] = None -@register_materializer("mtm_match_results") -def materialize_mtm_match_results( - data: List[TemplateMatchResult], - path: str, - filemanager, - backends: Union[str, List[str]], - backend_kwargs: dict = None, - spec=None, - context=None, - extra_inputs: dict | None = None, -) -> str: - """Materialize MTM match results as analysis-ready CSV with confidence analysis. - - Args: - data: List of template match results - path: Output path for results - filemanager: FileManager instance for I/O operations - backends: Single backend string or list of backends to save to - backend_kwargs: Dict mapping backend names to their kwargs - - Returns: - Path to the saved results file (first backend in list) - """ - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - - if backend_kwargs is None: - backend_kwargs = {} - - csv_path = _generate_output_path(path, "_mtm_matches.csv", ".csv") - - rows = [] - for result in data: - slice_idx = result.slice_index - - # Process all matches for this slice - # MTM hits format: [label, bbox, score] where bbox is (x, y, width, height) - for i, match in enumerate(result.matches): - if len(match) >= 3: - template_label, bbox, score = match[0], match[1], match[2] - x, y, w, h = bbox if len(bbox) >= 4 else (0, 0, 0, 0) - - rows.append({ - 'slice_index': slice_idx, - 'match_id': f"slice_{slice_idx}_match_{i}", - 'bbox_x': x, - 'bbox_y': y, - 'bbox_width': w, - 'bbox_height': h, - 'confidence_score': score, - 'template_name': template_label, - 'is_best_match': (match == result.best_match), - 'was_cropped': result.crop_bbox is not None - }) - else: - # Handle malformed match data - rows.append({ - 'slice_index': slice_idx, - 'match_id': f"slice_{slice_idx}_match_{i}", - 'bbox_x': 0, - 'bbox_y': 0, - 'bbox_width': 0, - 'bbox_height': 0, - 'confidence_score': 0.0, - 'template_name': 'malformed_match', - 'is_best_match': False, - 'was_cropped': result.crop_bbox is not None - }) - - if rows: - df = pd.DataFrame(rows) - - # Add analysis columns - if len(df) > 0 and 'confidence_score' in df.columns: - df['high_confidence'] = df['confidence_score'] > 0.8 - - # Only create quartiles if we have enough unique values - unique_scores = df['confidence_score'].nunique() - if unique_scores >= 4: - try: - df['confidence_quartile'] = pd.qcut(df['confidence_score'], 4, labels=['Q1', 'Q2', 'Q3', 'Q4'], duplicates='drop') - except ValueError: - # Fallback to simple binning if qcut fails - df['confidence_quartile'] = pd.cut(df['confidence_score'], 4, labels=['Q1', 'Q2', 'Q3', 'Q4']) - else: - # Not enough unique values for quartiles, use simple high/low classification - df['confidence_quartile'] = df['confidence_score'].apply(lambda x: 'High' if x > 0.8 else 'Low') - - # Add spatial clustering if we have position data - if 'bbox_x' in df.columns and 'bbox_y' in df.columns: - try: - from sklearn.cluster import KMeans - if len(df) >= 3: - coords = df[['bbox_x', 'bbox_y']].values - n_clusters = min(3, len(df)) - kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10) - df['spatial_cluster'] = kmeans.fit_predict(coords) - else: - df['spatial_cluster'] = 0 - except ImportError: - df['spatial_cluster'] = 0 - - csv_content = df.to_csv(index=False) - - # Save to all backends - for backend in backends: - # Ensure output directory exists for disk backend - if backend == Backend.DISK.value: - filemanager.ensure_directory(Path(csv_path).parent, backend) - - # Get backend-specific kwargs - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(csv_content, csv_path, backend, **kwargs) - - return csv_path - - @numpy_func -@special_outputs(("match_results", materializer_spec("mtm_match_results"))) +@special_outputs(( + "match_results", + MaterializationSpec( + CsvOptions(filename_suffix="_mtm_matches.csv", fields=["slice_index"], row_unpacker=_mtm_row_unpacker) + ), +)) def multi_template_crop_reference_channel( image_stack: np.ndarray, template_path: str, @@ -326,7 +255,12 @@ def multi_template_crop_reference_channel( @numpy_func -@special_outputs(("match_results", materializer_spec("mtm_match_results"))) +@special_outputs(( + "match_results", + MaterializationSpec( + CsvOptions(filename_suffix="_mtm_matches.csv", fields=["slice_index"], row_unpacker=_mtm_row_unpacker) + ), +)) def multi_template_crop_subset( image_stack: np.ndarray, template_path: str, @@ -475,7 +409,12 @@ def multi_template_crop_subset( @numpy_func -@special_outputs(("match_results", materializer_spec("mtm_match_results"))) +@special_outputs(( + "match_results", + MaterializationSpec( + CsvOptions(filename_suffix="_mtm_matches.csv", fields=["slice_index"], row_unpacker=_mtm_row_unpacker) + ), +)) def multi_template_crop( image_stack: np.ndarray, template_path: str, diff --git a/openhcs/processing/backends/analysis/skan_axon_analysis.py b/openhcs/processing/backends/analysis/skan_axon_analysis.py index 21c94afe1..f57b7e2d4 100644 --- a/openhcs/processing/backends/analysis/skan_axon_analysis.py +++ b/openhcs/processing/backends/analysis/skan_axon_analysis.py @@ -17,8 +17,7 @@ # OpenHCS imports from openhcs.core.memory import numpy as numpy_func from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import register_materializer, materializer_spec, tiff_stack_materializer -from openhcs.processing.materialization.core import _generate_output_path +from openhcs.processing.materialization import MaterializationSpec, CsvOptions, JsonOptions, ROIOptions, TiffStackOptions from polystore.roi import ROI logger = logging.getLogger(__name__) @@ -45,284 +44,26 @@ class AnalysisDimension(Enum): THREE_D = "3d" -@register_materializer("axon_analysis_skan") -def materialize_axon_analysis( - axon_analysis_data: Dict[str, Any], - path: str, - filemanager, - backends, - backend_kwargs: dict = None, - spec=None, - context=None, - extra_inputs: dict | None = None, -) -> str: - """ - Materialize axon analysis results to disk using filemanager. - - Creates multiple output files: - - CSV file with detailed branch data - - JSON file with summary metrics and metadata - - Optional: Excel file with multiple sheets - - Args: - axon_analysis_data: The axon analysis results dictionary - path: Base path for output files (from special output path) - filemanager: FileManager instance for consistent I/O - backends: Single backend string or list of backends to save to - backend_kwargs: Dict mapping backend names to their kwargs - - Returns: - str: Path to the primary output file (JSON summary) - """ - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - - if backend_kwargs is None: - backend_kwargs = {} - - logger.info(f"🔬 SKAN_MATERIALIZE: Called with path={path}, backends={backends}, data_keys={list(axon_analysis_data.keys()) if axon_analysis_data else 'None'}") - import json - from openhcs.constants.constants import Backend - - # Generate output file paths based on the input path - # Replace extension properly (handles .pkl, .roi.zip, or any extension) - base_path = _generate_output_path(path, "", "") - json_path = f"{base_path}.json" - csv_path = f"{base_path}_branches.csv" - parent_dir = Path(base_path).parent - - # 1. Prepare summary and metadata as JSON (primary output) - summary_data = { - 'analysis_type': 'axon_skeleton_analysis', - 'summary': axon_analysis_data['summary'], - 'metadata': axon_analysis_data['metadata'] - } - json_content = json.dumps(summary_data, indent=2, default=str) - - # 2. Prepare detailed branch data as CSV - branch_df = pd.DataFrame(axon_analysis_data['branch_data']) - csv_content = branch_df.to_csv(index=False) - - # 3. Save to all backends - for backend in backends: - # Ensure output directory exists for disk backend - if backend == Backend.DISK.value: - filemanager.ensure_directory(str(parent_dir), backend) - - # Get backend-specific kwargs - kwargs = backend_kwargs.get(backend, {}) - - # Get backend instance to check capabilities (polymorphic dispatch) - backend_instance = filemanager._get_backend(backend) - - # Only check exists/delete for backends that support filesystem operations - if backend_instance.requires_filesystem_validation: - # Storage backend - check and delete if exists - if filemanager.exists(json_path, backend): - filemanager.delete(json_path, backend) - if filemanager.exists(csv_path, backend): - filemanager.delete(csv_path, backend) - - # Save JSON and CSV (works for all backends) - filemanager.save(json_content, json_path, backend, **kwargs) - filemanager.save(csv_content, csv_path, backend, **kwargs) - - # 4. Optional: Create Excel file with multiple sheets (using direct file I/O for Excel) - # Note: Excel files require actual file paths, not compatible with OMERO backend - if kwargs.get('create_excel', True) and backend == Backend.DISK.value: - excel_path = str(parent_dir / f"{base_path}_complete.xlsx") - # Remove existing file if it exists - if Path(excel_path).exists(): - Path(excel_path).unlink() - with pd.ExcelWriter(excel_path, engine='openpyxl') as writer: - # Branch data sheet - branch_df.to_excel(writer, sheet_name='Branch_Data', index=False) - - # Summary sheet - summary_df = pd.DataFrame([axon_analysis_data['summary']]) - summary_df.to_excel(writer, sheet_name='Summary', index=False) - - # Metadata sheet - metadata_df = pd.DataFrame([axon_analysis_data['metadata']]) - metadata_df.to_excel(writer, sheet_name='Metadata', index=False) - - logger.info(f"Created Excel file: {excel_path}") - - # 4. Log materialization - logger.info("Materialized axon analysis:") - logger.info(f" - Summary: {json_path}") - logger.info(f" - Branch data: {csv_path}") - - return json_path - - -def materialize_skeleton_visualizations(data: List[np.ndarray], path: str, filemanager, backends, backend_kwargs: dict = None) -> str: - """Materialize skeleton visualizations as individual TIFF files (legacy).""" - - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - - if backend_kwargs is None: - backend_kwargs = {} - - # Generate output file paths based on the input path - base_path = _generate_output_path(path, "", "") - parent_dir = Path(base_path).parent - - # Check if data is None or empty (handle both None and empty arrays) - if data is None or (isinstance(data, np.ndarray) and data.size == 0): - # Create empty summary file to indicate no visualizations were generated - summary_path = f"{base_path}_skeleton_summary.txt" - summary_content = "No skeleton visualizations generated (return_skeleton_visualizations=False)\n" - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(summary_content, summary_path, backend, **kwargs) - return summary_path - - # Save each visualization as a separate TIFF file - for i, visualization in enumerate(data): - viz_filename = f"{base_path}_slice_{i:03d}.tif" - - # Convert visualization to uint8 for tracing visualizations (standard format) - if visualization.dtype != np.uint8: - # Normalize to uint8 range if needed - if visualization.max() <= 1.0: - viz_uint8 = (visualization * 255).astype(np.uint8) - else: - viz_uint8 = visualization.astype(np.uint8) - else: - viz_uint8 = visualization - - # Save to all backends - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(viz_uint8, viz_filename, backend, **kwargs) - - # Return summary path - summary_path = f"{base_path}_skeleton_summary.txt" - summary_content = f"Skeleton visualizations saved: {len(data)} files\n" - summary_content += f"Base filename pattern: {Path(base_path).name}_slice_XXX.tif\n" - summary_content += f"Visualization dtype: {data[0].dtype}\n" - summary_content += f"Visualization shape: {data[0].shape}\n" - - # Save summary to all backends - from openhcs.constants.constants import Backend - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(summary_content, summary_path, backend, **kwargs) - - return summary_path - - -@register_materializer("skeleton_rois_skan") -def materialize_skeleton_rois( - skeleton_mask, - path: str, - filemanager, - backends, - backend_kwargs: dict = None, - spec=None, - context=None, - extra_inputs: dict | None = None, -) -> str: - """ - Materialize skeleton mask as label image AND ROI ZIP files. - - Converts binary skeleton mask to: - 1. Label image where each branch has unique ID (for streaming/visualization) - 2. Polyline ROIs (for ImageJ/Fiji compatibility) - - Args: - skeleton_mask: Binary skeleton array (Z, Y, X) or list of arrays with skeleton pixels - path: Base path for output files - filemanager: FileManager instance - backends: Backend(s) to save to - backend_kwargs: Backend-specific kwargs - - Returns: - str: Path to the saved label image (primary output for streaming) - """ - # Normalize backends to list - if isinstance(backends, str): - backends = [backends] - - if backend_kwargs is None: - backend_kwargs = {} - - # Handle data that comes as a list (from multiple items/slices) - if isinstance(skeleton_mask, list): - if len(skeleton_mask) == 0: - skeleton_mask = np.zeros((0, 0, 0), dtype=bool) - elif len(skeleton_mask) == 1: - skeleton_mask = skeleton_mask[0] - else: - # Stack multiple masks into 3D array - skeleton_mask = np.stack(skeleton_mask, axis=0) - - logger.info(f"🔬 SKELETON_MATERIALIZE: Called with path={path}, mask_shape={skeleton_mask.shape if hasattr(skeleton_mask, 'shape') and skeleton_mask.size > 0 else 'empty'}, backends={backends}") - - # Check if skeleton mask is empty (return_skeleton_mask=False) - if not hasattr(skeleton_mask, 'size') or skeleton_mask.size == 0: - logger.info("🔬 SKELETON_MATERIALIZE: No skeleton mask to materialize (return_skeleton_mask=False)") - # Create empty summary file - base_path = path.replace('.pkl', '').replace('.roi.zip', '').replace('.tif', '') - summary_path = f"{base_path}_skeleton_summary.txt" - summary_content = "No skeleton mask generated (return_skeleton_mask=False)\n" - for backend in backends: - filemanager.save(summary_content, summary_path, backend) - return summary_path - - # Convert skeleton mask to label image (each branch gets unique ID) - skeleton_labels = _skeleton_mask_to_labels(skeleton_mask) - num_labels = int(skeleton_labels.max()) - logger.info(f"🔬 SKELETON_MATERIALIZE: Converted skeleton mask to label image with {num_labels} branches") - - # Also convert to ROIs for ImageJ/Fiji compatibility - skeleton_rois = _skeleton_mask_to_rois(skeleton_mask) - logger.info(f"🔬 SKELETON_MATERIALIZE: Converted skeleton mask to {len(skeleton_rois)} ROIs for ImageJ") - - # Generate output paths - base_path = path.replace('.pkl', '').replace('.roi.zip', '').replace('.tif', '') - labels_path = f"{base_path}_skeleton_labels.tif" - roi_path = f"{base_path}_skeleton.roi.zip" - - # Save label image (primary output for streaming) - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(skeleton_labels, labels_path, backend, **kwargs) - logger.info(f"🔬 SKELETON_MATERIALIZE: Saved skeleton labels to {labels_path} ({backend})") - - # Save ROIs for ImageJ/Fiji - if skeleton_rois: - for backend in backends: - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(skeleton_rois, roi_path, backend, **kwargs) - logger.info(f"🔬 SKELETON_MATERIALIZE: Saved {len(skeleton_rois)} skeleton ROIs to {roi_path} ({backend})") - - # Save summary - summary_path = f"{base_path}_skeleton_summary.txt" - summary_content = f"Skeleton branches: {num_labels}\n" - summary_content += f"Skeleton shape: {skeleton_mask.shape}\n" - summary_content += f"Label image: {labels_path}\n" - if skeleton_rois: - summary_content += f"ROI file: {roi_path}\n" - - for backend in backends: - filemanager.save(summary_content, summary_path, backend) - - # Return label image path (this is what gets streamed) - return labels_path - @special_outputs( - ("axon_analysis", materializer_spec("axon_analysis_skan")), - ("skeleton_visualizations", tiff_stack_materializer( - normalize_uint8=True, - summary_suffix="_skeleton_summary.txt" - )), - ("skeleton_masks", materializer_spec("skeleton_rois_skan")) # Mask output gets converted to ROIs + ( + "axon_analysis", + MaterializationSpec( + JsonOptions(filename_suffix=".json"), + CsvOptions(source="branch_data", filename_suffix="_branches.csv"), + primary=0, + ), + ), + ( + "skeleton_visualizations", + MaterializationSpec( + TiffStackOptions( + normalize_uint8=True, + summary_suffix="_skeleton_summary.txt", + ) + ), + ), + ("skeleton_masks", MaterializationSpec(ROIOptions())), ) @numpy_func def skan_axon_skeletonize_and_analyze( @@ -423,8 +164,12 @@ def skan_axon_skeletonize_and_analyze( ) skeleton_visualizations.append(visualization) - # Step 7: Return skeleton mask if requested (materializer will convert to ROIs) - skeleton_mask_output = skeleton_stack if return_skeleton_mask else np.zeros((0, 0, 0), dtype=bool) + # Step 7: Return skeleton mask if requested (converted to per-branch labels for ROI writer) + skeleton_mask_output = ( + _skeleton_mask_to_labels(skeleton_stack) + if return_skeleton_mask + else np.zeros((0, 0, 0), dtype=np.uint16) + ) # Step 8: Compile comprehensive results results = _compile_analysis_results( @@ -877,7 +622,8 @@ def _compile_analysis_results(branch_data, skeleton_stack, binary_stack, image_s # Combine all results results = { 'summary': {**summary, **segmentation_metrics}, - 'branch_data': branch_data.to_dict('list') if len(branch_data) > 0 else {}, + # Writer-based materialization expects list-of-dicts for tabular CSV. + 'branch_data': branch_data.to_dict('records') if len(branch_data) > 0 else [], 'metadata': { 'analysis_type': analysis_type, 'voxel_spacing': voxel_spacing, diff --git a/openhcs/processing/backends/lib_registry/unified_registry.py b/openhcs/processing/backends/lib_registry/unified_registry.py index 0a5f2b983..1aa8dcc6a 100644 --- a/openhcs/processing/backends/lib_registry/unified_registry.py +++ b/openhcs/processing/backends/lib_registry/unified_registry.py @@ -113,6 +113,19 @@ def get_registry_name(self) -> str: """ return self.registry.library_name + def get(self, key: str, default=None): + """ + Dict-like get method for compatibility with code expecting dict-like access. + + Args: + key: Attribute name to retrieve + default: Default value if attribute doesn't exist + + Returns: + Attribute value or default + """ + return getattr(self, key, default) + @@ -224,6 +237,9 @@ def apply_contract_wrapper(self, func: Callable, contract: ProcessingContract) - # If nothing to inject, return original function if not params_to_add: + # Still brand the callable as Enableable metadata. + from python_introspect import mark_enableable + mark_enableable(func, enabled_default=True) return func # Build new parameter list (insert before **kwargs) @@ -237,10 +253,28 @@ def apply_contract_wrapper(self, func: Callable, contract: ProcessingContract) - # Create wrapper @wraps(func) def wrapper(image, *args, **kwargs): + # Extract injectable params and set them as attributes on func for param_name, _, _ in injectable_params: if param_name in kwargs: setattr(func, param_name, kwargs[param_name]) - return contract.execute(self, func, image, *args, **kwargs) + + # Populate missing injectable params with their defaults from the signature + # This is critical for internal calls between OpenHCS functions where + # injectable params may not be explicitly passed (e.g., create_projection calling max_projection) + from python_introspect import SignatureAnalyzer + sig_params = SignatureAnalyzer.analyze(wrapper) + for param_name, _, _ in injectable_params: + if param_name not in kwargs and param_name in sig_params: + default_value = sig_params[param_name].default_value + if default_value is not inspect.Parameter.empty: + kwargs[param_name] = default_value + + # Filter injectable params from kwargs, EXCEPT dtype_config which needs to + # flow through to ArrayBridge's dtype_wrapper for conversion logic + params_to_filter = {name for name, _, _ in injectable_params if name != 'dtype_config'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k not in params_to_filter} + + return contract.execute(self, func, image, *args, **filtered_kwargs) # Set defaults and signature for param_name, default_value, _ in injectable_params: @@ -255,6 +289,11 @@ def wrapper(image, *args, **kwargs): if hasattr(func, '__processing_contract__'): wrapper.__processing_contract__ = func.__processing_contract__ + # Nominal enable semantics: decorated callables are Enableable. + # (Enableable is metadata only; enabled remains a kw-only param for call sites.) + from python_introspect import mark_enableable + mark_enableable(wrapper, enabled_default=True) + return wrapper def _inject_optional_dataclass_params(self, func: Callable) -> Callable: @@ -658,9 +697,18 @@ def create_library_adapter(self, original_func: Callable, contract: ProcessingCo # Get original signature to preserve it original_sig = inspect.signature(original_func) + # Wrap external library functions with ArrayBridge decorator for dtype handling + arraybridge_wrapped_func = original_func + if self.MEMORY_TYPE is not None: + from arraybridge.decorators import _create_dtype_wrapper + from arraybridge.types import MemoryType as ABMemoryType + # Map memory type string to ArrayBridge MemoryType enum + mem_type = ABMemoryType(self.MEMORY_TYPE) + arraybridge_wrapped_func = _create_dtype_wrapper(original_func, mem_type, func_name) + def adapter(image, *args, **kwargs): processed_image = self._preprocess_input(image, func_name) - result = contract.execute(self, original_func, processed_image, *args, **kwargs) + result = contract.execute(self, arraybridge_wrapped_func, processed_image, *args, **kwargs) return self._postprocess_output(result, image, func_name) # Apply wraps and preserve signature diff --git a/openhcs/processing/backends/processors/tensorflow_processor.py b/openhcs/processing/backends/processors/tensorflow_processor.py index 607959c0c..1327dbe5a 100644 --- a/openhcs/processing/backends/processors/tensorflow_processor.py +++ b/openhcs/processing/backends/processors/tensorflow_processor.py @@ -9,12 +9,12 @@ - Clause 88 — No Inferred Capabilities: Explicit TensorFlow dependency - Clause 106-A — Declared Memory Types: All methods specify TensorFlow tensors """ -from __future__ import annotations +from __future__ import annotations import logging from typing import Any, List, Optional, Tuple -import pkg_resources +from packaging.version import parse as parse_version from openhcs.core.memory import tensorflow as tensorflow_func from openhcs.core.lazy_gpu_imports import tf @@ -25,8 +25,8 @@ # Check TensorFlow version for DLPack compatibility if available if tf is not None: try: - tf_version = pkg_resources.parse_version(tf.__version__) - min_version = pkg_resources.parse_version("2.12.0") + tf_version = parse_version(tf.__version__) + min_version = parse_version("2.12.0") if tf_version < min_version: TENSORFLOW_ERROR = ( diff --git a/openhcs/processing/custom_functions/templates.py b/openhcs/processing/custom_functions/templates.py index 27cb94f90..495d1e4a5 100644 --- a/openhcs/processing/custom_functions/templates.py +++ b/openhcs/processing/custom_functions/templates.py @@ -107,15 +107,15 @@ def my_custom_function(image, scale: float = 1.0, offset: float = 0.0): # 1. Import the decorators and materializers: # # from openhcs.core.pipeline.function_contracts import special_outputs -# from openhcs.processing.materialization import csv_materializer +# from openhcs.processing.materialization import MaterializationSpec, CsvOptions, JsonOptions # # 2. Declare outputs with @special_outputs: # # @numpy -# @special_outputs(("measurements", csv_materializer( +# @special_outputs(("measurements", MaterializationSpec(CsvOptions( # fields=["slice_index", "mean", "std"], # analysis_type="intensity_stats" -# ))) +# )))) # def analyze_intensity(image, threshold: float = 0.5): # results = [] # for i, slice_data in enumerate(image): @@ -128,9 +128,9 @@ def my_custom_function(image, scale: float = 1.0, offset: float = 0.0): # # 3. Available materializers: # -# csv_materializer(fields=[...], analysis_type="...") - CSV file -# json_materializer(fields=[...], analysis_type="...") - JSON file -# dual_materializer(fields=[...], summary_fields=[...]) - Both CSV + JSON +# MaterializationSpec(CsvOptions(...)) - CSV file +# MaterializationSpec(JsonOptions(...)) - JSON file +# MaterializationSpec(JsonOptions(...), CsvOptions(...), primary=0) - Both JSON + CSV # # ============================================================================= """ @@ -311,7 +311,7 @@ def my_custom_function(image, radius: float = 2.0): NUMPY_ANALYSIS_TEMPLATE = """from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import csv_materializer +from openhcs.processing.materialization import CsvOptions, MaterializationSpec from dataclasses import dataclass from typing import List, Tuple import numpy as np @@ -326,10 +326,10 @@ class AnalysisResult: @numpy -@special_outputs(("analysis_results", csv_materializer( +@special_outputs(("analysis_results", MaterializationSpec(CsvOptions( fields=["slice_index", "measurement", "count"], - analysis_type="slice_analysis" -))) + filename_suffix=".csv" +)))) def my_analysis_function(image, threshold: float = 0.5) -> Tuple[np.ndarray, List[AnalysisResult]]: \"\"\" Analysis function that produces both processed image AND structured results. @@ -345,7 +345,7 @@ def my_analysis_function(image, threshold: float = 0.5) -> Tuple[np.ndarray, Lis Notes: - @special_outputs declares that this function produces analysis data - - The csv_materializer auto-converts AnalysisResult fields to CSV columns + - CsvOptions auto-converts AnalysisResult fields to CSV columns - Return is ALWAYS a tuple: (image, special_output_1, special_output_2, ...) \"\"\" results = [] @@ -372,7 +372,7 @@ def my_analysis_function(image, threshold: float = 0.5) -> Tuple[np.ndarray, Lis NUMPY_DUAL_OUTPUT_TEMPLATE = """from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import csv_materializer, dual_materializer +from openhcs.processing.materialization import CsvOptions, JsonOptions, MaterializationSpec from dataclasses import dataclass from typing import List, Tuple, Dict, Any import numpy as np @@ -388,7 +388,7 @@ class CellMeasurement: intensity: float -@dataclass +@dataclass class SliceSummary: \"\"\"Per-slice summary statistics.\"\"\" slice_index: int @@ -399,14 +399,11 @@ class SliceSummary: @numpy @special_outputs( - ("cell_measurements", csv_materializer( - filename_suffix="_cells.csv", - analysis_type="cell_measurements" - )), - ("slice_summaries", dual_materializer( - summary_fields=["slice_index", "cell_count"], - fields=["slice_index", "cell_count", "total_area", "mean_intensity"], - analysis_type="slice_summaries" + ("cell_measurements", MaterializationSpec(CsvOptions(filename_suffix="_cells.csv"))), + ("slice_summaries", MaterializationSpec( + JsonOptions(filename_suffix=".json", wrap_list=True), + CsvOptions(filename_suffix="_details.csv"), + primary=0, )) ) def analyze_cells( diff --git a/openhcs/processing/materialization/__init__.py b/openhcs/processing/materialization/__init__.py index 1c3b3ac2f..1568c765d 100644 --- a/openhcs/processing/materialization/__init__.py +++ b/openhcs/processing/materialization/__init__.py @@ -1,50 +1,50 @@ -""" -Materialization framework for analysis outputs. - -Provides composable, declarative materialization functions that transform -analysis results (dataclasses, dicts, arrays) into serialized formats -(CSV, JSON, ROI archives) via the VFS. - -Architecture: - - MaterializationSpec: Declarative spec for how to serialize a type - - MaterializationRegistry + materialize(): Registry + dispatcher for specs - - Built-in spec builders for common patterns (csv/json/dual/roi/tiff) - -Usage: - from openhcs.processing.materialization import csv_materializer, json_materializer - - @special_outputs(("cell_counts", csv_materializer( - fields=["slice_index", "cell_count", "avg_area"], - filename_suffix="_cell_counts.csv" - ))) - def count_cells(image): - ... -""" +"""Materialization public API (writer-based).""" +from openhcs.processing.materialization.constants import MaterializationFormat, WriteMode from openhcs.processing.materialization.core import ( + BackendSaver, + MaterializationContext, MaterializationSpec, - MaterializationRegistry, - register_materializer, + Output, + PathHelper, materialize, - csv_materializer, - json_materializer, - dual_materializer, - roi_zip_materializer, - regionprops_materializer, - tiff_stack_materializer, - materializer_spec, +) +from openhcs.processing.materialization.options import ( + CsvOptions, + FileOutputOptions, + JsonOptions, + ROIOptions, + TextOptions, + TiffStackOptions, +) +from openhcs.processing.materialization.presets import ( + csv_only, + json_and_csv, + json_only, + roi_zip, + text_only, + tiff_stack, ) __all__ = [ + "MaterializationFormat", + "WriteMode", "MaterializationSpec", - "MaterializationRegistry", - "register_materializer", + "MaterializationContext", + "Output", + "PathHelper", + "BackendSaver", "materialize", - "csv_materializer", - "json_materializer", - "dual_materializer", - "roi_zip_materializer", - "regionprops_materializer", - "tiff_stack_materializer", - "materializer_spec", + "FileOutputOptions", + "CsvOptions", + "JsonOptions", + "ROIOptions", + "TiffStackOptions", + "TextOptions", + "json_only", + "csv_only", + "json_and_csv", + "roi_zip", + "tiff_stack", + "text_only", ] diff --git a/openhcs/processing/materialization/constants.py b/openhcs/processing/materialization/constants.py new file mode 100644 index 000000000..78f8c31f1 --- /dev/null +++ b/openhcs/processing/materialization/constants.py @@ -0,0 +1,22 @@ +"""Constants for the materialization system (greenfield).""" + +from __future__ import annotations + +from enum import Enum + + +class MaterializationFormat(str, Enum): + """Built-in output formats (writer keys).""" + + CSV = "csv" + JSON = "json" + ROI_ZIP = "roi_zip" + TIFF_STACK = "tiff_stack" + TEXT = "text" + + +class WriteMode(str, Enum): + """Overwrite/delete semantics for materialization writes.""" + + OVERWRITE = "overwrite" + ERROR = "error" diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py index 94128e6d5..14b94e4cc 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/processing/materialization/core.py @@ -1,975 +1,521 @@ -""" -Core materialization framework. +"""Materialization core (writer-based, greenfield). -Provides a single spec + registry + dispatcher abstraction for analysis materialization. +Key idea: the abstraction boundary is the output *format* (writers), not per-analysis handlers. """ from __future__ import annotations import json import logging -from dataclasses import dataclass, field, fields, is_dataclass +from dataclasses import dataclass from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence - -import pandas as pd - -from openhcs.constants.constants import Backend +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from openhcs.processing.materialization.constants import MaterializationFormat, WriteMode +from openhcs.processing.materialization.options import ( + CsvOptions, + FileOutputOptions, + JsonOptions, + ROIOptions, + TextOptions, + TiffStackOptions, +) +from openhcs.processing.materialization.utils import ( + discover_array_fields, + expand_array_field, + extract_fields, +) logger = logging.getLogger(__name__) @dataclass(frozen=True) -class MaterializationSpec: - """ - Declarative materialization spec. +class Output: + path: str + content: Any - handler: Registered handler name in MaterializationRegistry. - options: Handler-specific options (serializable). - allowed_backends: Optional allowlist of backend names. - """ - handler: str - options: Dict[str, Any] = field(default_factory=dict) - allowed_backends: Optional[List[str]] = None +def _resolve_source(value: Any, source: Optional[str]) -> Any: + if not source: + return value -@dataclass(frozen=True) -class MaterializationHandler: - name: str - func: Callable[..., str] - requires_arbitrary_files: bool = True + cur = value + for part in source.split("."): + if isinstance(cur, dict): + cur = cur[part] + else: + cur = getattr(cur, part) + return cur -class MaterializationRegistry: - _handlers: Dict[str, MaterializationHandler] = {} +def _output_path(ctx: "MaterializationContext", options: FileOutputOptions) -> str: + return ctx.paths(options).with_suffix(options.filename_suffix) - @classmethod - def register( - cls, - name: str, - func: Callable[..., str], - *, - requires_arbitrary_files: bool = True - ) -> None: - if name in cls._handlers: - raise ValueError(f"Materialization handler already registered: {name}") - cls._handlers[name] = MaterializationHandler( - name=name, - func=func, - requires_arbitrary_files=requires_arbitrary_files - ) - @classmethod - def get(cls, name: str) -> MaterializationHandler: - if name not in cls._handlers: - raise KeyError(f"Unknown materialization handler: {name}") - return cls._handlers[name] +def _select_payload(data: Any, options: Any) -> Any: + return _resolve_source(data, getattr(options, "source", None)) -def register_materializer( - name: str, - *, - requires_arbitrary_files: bool = True -) -> Callable[[Callable[..., str]], Callable[..., str]]: - def decorator(func: Callable[..., str]) -> Callable[..., str]: - MaterializationRegistry.register( - name, - func, - requires_arbitrary_files=requires_arbitrary_files - ) - func.__materialization_handler__ = name - return func - return decorator +def _is_empty(value: Any) -> bool: + if value is None: + return True + size = getattr(value, "size", None) + if isinstance(size, int) and size == 0: + return True + try: + return len(value) == 0 # type: ignore[arg-type] + except Exception: + return False -def _normalize_backends(backends: Sequence[str] | str) -> List[str]: - if isinstance(backends, str): - return [backends] - return list(backends) +def _as_sequence(value: Any) -> list[Any]: + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + return [value] -def _extract_fields(item: Any, field_names: Optional[List[str]] = None) -> Dict[str, Any]: - """Extract fields from dataclass or dict.""" - if is_dataclass(item): - if field_names: - return {f: getattr(item, f, None) for f in field_names if hasattr(item, f)} - return {f.name: getattr(item, f.name) for f in fields(item)} - if isinstance(item, dict): - if field_names: - return {f: item.get(f) for f in field_names if f in item} - return item - return {"value": item} +class PathHelper: + """Path helper parameterized by output options.""" + def __init__(self, base_path: str, options: FileOutputOptions): + self.base_path = self._strip_path(base_path, options) + self.parent = self.base_path.parent + self.name = self.base_path.name -def _strip_known_suffixes(path: str, *, strip_roi: bool = False, strip_pkl: bool = True) -> Path: - """ - Strip known suffixes from a path while preserving compound suffix semantics. + @staticmethod + def _strip_path(path: str, options: FileOutputOptions) -> Path: + p = Path(path) + name = p.name - Note: Path.stem only strips the last suffix (e.g., ".zip"), which breaks compound - suffixes like ".roi.zip" (it leaves a dangling ".roi"). We treat ".roi.zip" as a - single compound suffix and remove it atomically. - """ - p = Path(path) - name = p.name + if name.endswith(".roi.zip"): + name = name[: -len(".roi.zip")] - # Strip compound suffixes first - if name.endswith(".roi.zip"): - name = name[: -len(".roi.zip")] + if options.strip_pkl and name.endswith(".pkl"): + name = name[: -len(".pkl")] + if options.strip_roi_suffix and name.endswith(".roi"): + name = name[: -len(".roi")] - # Strip common single suffixes - if strip_pkl and name.endswith(".pkl"): - name = name[: -len(".pkl")] - if strip_roi and name.endswith(".roi"): - name = name[: -len(".roi")] + return p.with_name(name) - return p.with_name(name) + def with_suffix(self, suffix: str) -> str: + return str(self.parent / f"{self.name}{suffix}") -def _generate_output_path( - base_path: str, - suffix: str, - default_ext: str, - *, - strip_roi: bool = False, - strip_pkl: bool = True -) -> str: - """Generate output path with proper suffix handling.""" - path = _strip_known_suffixes(base_path, strip_roi=strip_roi, strip_pkl=strip_pkl) - parent = path.parent - if suffix: - return str(parent / f"{path.name}{suffix}") - return str(parent / f"{path.name}{default_ext}") +class BackendSaver: + """Centralized multi-backend saving.""" + def __init__( + self, + backends: list[str], + filemanager: Any, + backend_kwargs: dict[str, dict[str, Any]] | None, + *, + write_mode: WriteMode, + ): + self.backends = backends + self.filemanager = filemanager + self.backend_kwargs = backend_kwargs or {} + self.write_mode = write_mode -def _prepare_output_path(filemanager, backend: str, output_path: str) -> None: - backend_instance = filemanager._get_backend(backend) - if backend_instance.requires_filesystem_validation: - filemanager.ensure_directory(str(Path(output_path).parent), backend) - if filemanager.exists(output_path, backend): - filemanager.delete(output_path, backend) + def save(self, content: Any, path: str) -> None: + for backend in self.backends: + self._prepare_path(backend, path) + kwargs = self.backend_kwargs.get(backend, {}) + self.filemanager.save(content, path, backend, **kwargs) + def _prepare_path(self, backend: str, path: str) -> None: + backend_instance = self.filemanager._get_backend(backend) + if not backend_instance.requires_filesystem_validation: + return -def _validate_backends( - spec: MaterializationSpec, - handler: MaterializationHandler, - backends: List[str], - filemanager -) -> None: - if spec.allowed_backends is not None: - disallowed = [b for b in backends if b not in spec.allowed_backends] - if disallowed: - raise ValueError( - f"Materialization handler '{handler.name}' does not allow backends: {disallowed}. " - f"Allowed: {spec.allowed_backends}" - ) - - if handler.requires_arbitrary_files: - for backend in backends: - backend_instance = filemanager._get_backend(backend) - if not backend_instance.supports_arbitrary_files: - raise ValueError( - f"Backend '{backend}' does not support arbitrary files for handler " - f"'{handler.name}'." - ) + self.filemanager.ensure_directory(str(Path(path).parent), backend) + if not self.filemanager.exists(path, backend): + return -def materialize( - spec: MaterializationSpec, - data: Any, - path: str, - filemanager, - backends: Sequence[str] | str, - backend_kwargs: Optional[Dict[str, Dict[str, Any]]] = None, - context: Any = None, - extra_inputs: Optional[Dict[str, Any]] = None, -) -> str: - handler = MaterializationRegistry.get(spec.handler) - normalized_backends = _normalize_backends(backends) - _validate_backends(spec, handler, normalized_backends, filemanager) - return handler.func( - data, - path, - filemanager, - normalized_backends, - backend_kwargs or {}, - spec, - context, - extra_inputs or {}, - ) + if self.write_mode == WriteMode.OVERWRITE: + self.filemanager.delete(path, backend) + return + if self.write_mode == WriteMode.ERROR: + raise FileExistsError(f"Refusing to overwrite existing path: {path} ({backend})") -# Built-in handlers + raise ValueError(f"Unknown WriteMode: {self.write_mode!r}") -@register_materializer("csv", requires_arbitrary_files=True) -def _materialize_csv( - data: List[Any], - path: str, - filemanager, - backends: List[str], - backend_kwargs: Dict[str, Dict[str, Any]], - spec: MaterializationSpec, - context: Any, - extra_inputs: Dict[str, Any], -) -> str: - options = spec.options - fields_opt = options.get("fields") - filename_suffix = options.get("filename_suffix", ".csv") - strip_roi = options.get("strip_roi_suffix", False) - output_path = _generate_output_path(path, filename_suffix, ".csv", strip_roi=strip_roi) - - rows = [] - for i, item in enumerate(data or []): - row = _extract_fields(item, fields_opt) - if "slice_index" not in row: - row["slice_index"] = i - rows.append(row) - - if rows: - df = pd.DataFrame(rows) - csv_content = df.to_csv(index=False) - for backend in backends: - _prepare_output_path(filemanager, backend, output_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(csv_content, output_path, backend, **kwargs) - - logger.info(f"Materialized {len(rows)} rows to {output_path}") - return output_path - - -@register_materializer("json", requires_arbitrary_files=True) -def _materialize_json( - data: List[Any], - path: str, - filemanager, - backends: List[str], - backend_kwargs: Dict[str, Dict[str, Any]], - spec: MaterializationSpec, - context: Any, - extra_inputs: Dict[str, Any], -) -> str: - options = spec.options - fields_opt = options.get("fields") - filename_suffix = options.get("filename_suffix", ".json") - analysis_type = options.get("analysis_type") - include_metadata = options.get("include_metadata", True) - strip_roi = options.get("strip_roi_suffix", False) - output_path = _generate_output_path(path, filename_suffix, ".json", strip_roi=strip_roi) - - results = [] - for i, item in enumerate(data or []): - record = _extract_fields(item, fields_opt) - if "slice_index" not in record: - record["slice_index"] = i - results.append(record) - - summary = { - "total_items": len(results), - "results": results - } - if include_metadata and analysis_type: - summary["analysis_type"] = analysis_type - - json_content = json.dumps(summary, indent=2, default=str) - for backend in backends: - _prepare_output_path(filemanager, backend, output_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(json_content, output_path, backend, **kwargs) - - logger.info(f"Materialized {len(results)} items to {output_path}") - return output_path - - -@register_materializer("dual", requires_arbitrary_files=True) -def _materialize_dual( - data: List[Any], - path: str, - filemanager, - backends: List[str], - backend_kwargs: Dict[str, Dict[str, Any]], - spec: MaterializationSpec, - context: Any, - extra_inputs: Dict[str, Any], -) -> str: - options = spec.options - fields_opt = options.get("fields") - summary_fields = options.get("summary_fields") - analysis_type = options.get("analysis_type") - include_metadata = options.get("include_metadata", True) - strip_roi = options.get("strip_roi_suffix", False) - base_path = _generate_output_path(path, "", "", strip_roi=strip_roi) - json_path = f"{base_path}.json" - csv_path = f"{base_path}_details.csv" - - rows = [] - for i, item in enumerate(data or []): - row = _extract_fields(item, fields_opt) - if "slice_index" not in row: - row["slice_index"] = i - rows.append(row) - - summary_data = [] - for i, item in enumerate(data or []): - record = _extract_fields(item, summary_fields or fields_opt) - if "slice_index" not in record: - record["slice_index"] = i - summary_data.append(record) - - summary = { - "total_items": len(summary_data), - "results": summary_data - } - if include_metadata and analysis_type: - summary["analysis_type"] = analysis_type - - for backend in backends: - _prepare_output_path(filemanager, backend, json_path) - kwargs = backend_kwargs.get(backend, {}) - if rows: - df = pd.DataFrame(rows) - filemanager.save(df.to_csv(index=False), csv_path, backend, **kwargs) - filemanager.save(json.dumps(summary, indent=2, default=str), json_path, backend, **kwargs) - - logger.info(f"Materialized {len(rows)} rows (dual format) to {json_path}") - return json_path - - -@register_materializer("tiff_stack", requires_arbitrary_files=True) -def _materialize_tiff_stack( - data: List[Any], - path: str, - filemanager, - backends: List[str], - backend_kwargs: Dict[str, Dict[str, Any]], - spec: MaterializationSpec, - context: Any, - extra_inputs: Dict[str, Any], -) -> str: - options = spec.options - normalize_uint8 = options.get("normalize_uint8", False) - summary_suffix = options.get("summary_suffix", "_summary.txt") - strip_roi = options.get("strip_roi_suffix", False) - - if not data: - summary_path = _generate_output_path(path, summary_suffix, ".txt", strip_roi=strip_roi) - summary_content = options.get( - "empty_summary", - "No images generated (empty data)\n" - ) - for backend in backends: - _prepare_output_path(filemanager, backend, summary_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(summary_content, summary_path, backend, **kwargs) - return summary_path - - base_path = _generate_output_path(path, "", "", strip_roi=strip_roi) - for i, arr in enumerate(data): - filename = f"{base_path}_slice_{i:03d}.tif" - out_arr = arr - if normalize_uint8: - if out_arr.dtype != "uint8": - max_val = getattr(out_arr, "max", lambda: 0)() - if max_val <= 1.0: - out_arr = (out_arr * 255).astype("uint8") - else: - out_arr = out_arr.astype("uint8") - - for backend in backends: - _prepare_output_path(filemanager, backend, filename) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(out_arr, filename, backend, **kwargs) - - summary_path = f"{base_path}{summary_suffix}" - summary_content = f"Images saved: {len(data)} files\n" - summary_content += f"Base filename pattern: {base_path}_slice_XXX.tif\n" - summary_content += f"Image dtype: {data[0].dtype}\n" - summary_content += f"Image shape: {data[0].shape}\n" - - for backend in backends: - _prepare_output_path(filemanager, backend, summary_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(summary_content, summary_path, backend, **kwargs) - - return summary_path - - -@register_materializer("roi_zip", requires_arbitrary_files=True) -def _materialize_roi_zip( - data: List[Any], - path: str, - filemanager, - backends: List[str], - backend_kwargs: Dict[str, Dict[str, Any]], - spec: MaterializationSpec, - context: Any, - extra_inputs: Dict[str, Any], -) -> str: - options = spec.options - min_area = options.get("min_area", 10) - extract_contours = options.get("extract_contours", True) - roi_suffix = options.get("roi_suffix", "_rois.roi.zip") - summary_suffix = options.get("summary_suffix", "_segmentation_summary.txt") - strip_roi = options.get("strip_roi_suffix", False) - - if not data: - summary_path = _generate_output_path(path, summary_suffix, ".txt", strip_roi=strip_roi) - summary_content = options.get( - "empty_summary", - "No segmentation masks generated (empty data)\n" + +@dataclass(frozen=True) +class MaterializationContext: + base_path: str + backends: list[str] + backend_kwargs: dict[str, dict[str, Any]] + filemanager: Any + extra_inputs: dict[str, Any] + context: Any = None + write_mode: WriteMode = WriteMode.OVERWRITE + + def paths(self, options: FileOutputOptions) -> PathHelper: + return PathHelper(self.base_path, options) + + @property + def saver(self) -> BackendSaver: + return BackendSaver( + self.backends, + self.filemanager, + self.backend_kwargs, + write_mode=self.write_mode, ) - for backend in backends: - _prepare_output_path(filemanager, backend, summary_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(summary_content, summary_path, backend, **kwargs) - return summary_path - from polystore.roi import extract_rois_from_labeled_mask - all_rois = [] - for mask in data: - rois = extract_rois_from_labeled_mask( - mask, - min_area=min_area, - extract_contours=extract_contours - ) - all_rois.extend(rois) +@dataclass(frozen=True) +class WriterSpec: + format: MaterializationFormat + options_type: type + write: Callable[[Any, Any, MaterializationContext], list[Output]] + primary_path: Callable[[list[Output]], str] - base_path = _generate_output_path(path, "", "", strip_roi=strip_roi) - roi_path = f"{base_path}{roi_suffix}" - if all_rois: - for backend in backends: - _prepare_output_path(filemanager, backend, roi_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(all_rois, roi_path, backend, **kwargs) - - summary_path = f"{base_path}{summary_suffix}" - summary_content = f"Segmentation ROIs: {len(all_rois)} cells\n" - summary_content += f"Z-planes: {len(data)}\n" - if all_rois: - summary_content += f"ROI file: {roi_path}\n" - else: - summary_content += "No ROIs extracted (all regions below min_area threshold)\n" +_WRITERS_BY_OPTIONS: Dict[type, WriterSpec] = {} - for backend in backends: - _prepare_output_path(filemanager, backend, summary_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(summary_content, summary_path, backend, **kwargs) - return summary_path +def writer_for( + options_type: type, + fmt: MaterializationFormat, + *, + primary_path: Optional[Callable[[list[Output]], str]] = None, +): + """Register a writer for a given options type. + This is intentionally metaprogramming-friendly: adding a new format is + defining one options dataclass and one function. + """ + + def decorator(fn: Callable[[Any, Any, MaterializationContext], list[Output]]): + if options_type in _WRITERS_BY_OPTIONS: + raise ValueError(f"Writer already registered for options type {options_type.__name__}") + _WRITERS_BY_OPTIONS[options_type] = WriterSpec( + format=fmt, + options_type=options_type, + write=fn, + primary_path=primary_path or (lambda outs: outs[0].path if outs else ""), + ) + return fn + + return decorator -def _coerce_jsonable(value: Any) -> Any: + +def _wants_tabular(options: Any) -> bool: + return bool( + getattr(options, "fields", None) + or getattr(options, "row_field", None) + or getattr(options, "row_unpacker", None) + or getattr(options, "row_columns", None) + ) + + +def _build_tabular_rows(data: Any, options: Any) -> list[dict[str, Any]]: + # pandas support (optional): treat DataFrame as rows. try: - import numpy as np + import pandas as pd - if isinstance(value, np.generic): - return value.item() - if isinstance(value, np.ndarray): - return value.tolist() - except Exception: + if isinstance(data, pd.DataFrame): + return data.to_dict(orient="records") + if isinstance(data, pd.Series): + return [data.to_dict()] + except ImportError: pass - return value + items = _as_sequence(data) + rows: list[dict[str, Any]] = [] -@register_materializer("regionprops", requires_arbitrary_files=True) -def _materialize_regionprops( - data: Any, - path: str, - filemanager, - backends: List[str], - backend_kwargs: Dict[str, Dict[str, Any]], - spec: MaterializationSpec, - context: Any, - extra_inputs: Dict[str, Any], -) -> str: - """ - Materialize region properties from labeled masks. + for idx, item in enumerate(items): + base_row = extract_fields(item, getattr(options, "fields", None)) + if "slice_index" not in base_row: + base_row["slice_index"] = idx - Input: - - data: labeled mask(s) as list[2D] or 3D ndarray (Z, Y, X) - - extra_inputs["intensity"]: optional aligned intensity slices to compute intensity metrics + row_unpacker = getattr(options, "row_unpacker", None) + if row_unpacker: + for exp_row in row_unpacker(item): + rows.append({**base_row, **exp_row}) + continue - Output: - - ROI zip archive (for Fiji/Napari) + CSV details + JSON summary - """ - import numpy as np - from skimage.measure import regionprops_table - from polystore.roi import ROI, extract_rois_from_labeled_mask - - options = spec.options - analysis_type = options.get("analysis_type", "regionprops") - min_area = int(options.get("min_area", 10)) - extract_contours = bool(options.get("extract_contours", True)) - roi_suffix = options.get("roi_suffix", "_rois.roi.zip") - details_suffix = options.get("details_suffix", "_details.csv") - json_suffix = options.get("json_suffix", ".json") - require_intensity = bool(options.get("require_intensity", False)) - - base_properties = list(options.get("properties") or ["label", "area", "perimeter", "centroid", "bbox"]) - intensity_properties = list(options.get("intensity_properties") or ["mean_intensity"]) - - # Ensure required properties for filtering + stable schema - for required in ("label", "area", "centroid", "bbox"): - if required not in base_properties: - base_properties.append(required) - - intensity = extra_inputs.get("intensity") - if intensity is None: - intensity = extra_inputs.get("intensity_slices") - - def _normalize_slices(obj: Any, *, name: str) -> List[np.ndarray]: - if obj is None: - return [] - if isinstance(obj, list): - return [np.asarray(x) for x in obj] - arr = np.asarray(obj) - if arr.ndim == 2: - return [arr] - if arr.ndim == 3: - return [arr[i] for i in range(arr.shape[0])] - raise ValueError(f"{name} must be a 2D/3D array or list of 2D arrays, got shape {arr.shape}") - - label_slices = _normalize_slices(data, name="labeled_mask") - intensity_slices = _normalize_slices(intensity, name="intensity") if intensity is not None else [] - - if require_intensity and not intensity_slices: - raise ValueError( - "regionprops materializer requires intensity input, but none was provided. " - "Pass extra_inputs['intensity'] (aligned to label slices) or set require_intensity=False." - ) - if intensity_slices and len(intensity_slices) != len(label_slices): - raise ValueError( - f"Intensity/label slice mismatch: intensity={len(intensity_slices)} slices, " - f"labels={len(label_slices)} slices." - ) + row_field = getattr(options, "row_field", None) + if row_field: + array_data = getattr(item, row_field) + rows.extend(expand_array_field(array_data, base_row, getattr(options, "row_columns", {}) or {})) + continue - base_path = _generate_output_path(path, "", "") - json_path = f"{base_path}{json_suffix}" - csv_path = f"{base_path}{details_suffix}" - roi_path = f"{base_path}{roi_suffix}" - - all_rois: List[ROI] = [] - rows: List[Dict[str, Any]] = [] - per_slice: List[Dict[str, Any]] = [] - - for z_idx, labels in enumerate(label_slices): - if labels.ndim != 2: - raise ValueError(f"Label slice {z_idx} must be 2D, got shape {labels.shape}") - if not np.issubdtype(labels.dtype, np.integer): - labels = labels.astype(np.int32) - - intensity_img = None - if intensity_slices: - intensity_img = intensity_slices[z_idx] - if intensity_img.ndim != 2: - raise ValueError(f"Intensity slice {z_idx} must be 2D, got shape {intensity_img.shape}") - if intensity_img.shape != labels.shape: - raise ValueError( - f"Intensity slice {z_idx} shape {intensity_img.shape} does not match " - f"label slice shape {labels.shape}." - ) + if array_fields := discover_array_fields(item): + primary_field = array_fields[0] + array_data = getattr(item, primary_field) + rows.extend(expand_array_field(array_data, base_row, {})) + continue - props = list(dict.fromkeys(base_properties + (intensity_properties if intensity_img is not None else []))) - table = regionprops_table( - labels, - intensity_image=intensity_img, - properties=props - ) + rows.append(base_row) - # Filter out tiny regions (match ROI extraction behavior) - areas = table.get("area") - keep_idx: List[int] = [] - if areas is not None: - keep_idx = [i for i, a in enumerate(areas) if float(a) >= min_area] + return rows + + +def _render_csv(data: Any, options: CsvOptions) -> str: + try: + import pandas as pd + + if isinstance(data, pd.DataFrame): + return data.to_csv(index=False) + + rows = _build_tabular_rows(data, options) + return pd.DataFrame(rows).to_csv(index=False) + except ImportError: + raise ImportError("CSV materialization requires pandas") + + +def _render_json(data: Any, options: JsonOptions) -> str: + # Make common OpenHCS outputs JSON-friendly: + # - dataclass -> dict + # - list[dataclass] -> list[dict] + # - list[dict] unchanged + # If the options look tabular, use the canonical tabular builder. + payload: Any + if _wants_tabular(options): + payload = _build_tabular_rows(data, options) + else: + seq = _as_sequence(data) + if len(seq) == 1 and seq[0] is data: + # single element (non-list input) + payload = extract_fields(data, getattr(options, "fields", None)) else: - keep_idx = list(range(len(table.get("label", [])))) - - labels_col = table.get("label", []) - kept_labels: List[int] = [int(labels_col[i]) for i in keep_idx] - - # Build rows from regionprops_table output - for i in keep_idx: - row: Dict[str, Any] = {"slice_index": z_idx} - for key, values in table.items(): - if values is None: - continue - norm_key = key.replace("-", "_") - row[norm_key] = _coerce_jsonable(values[i]) - rows.append(row) - - # Per-slice summary - slice_summary: Dict[str, Any] = {"slice_index": z_idx, "region_count": len(keep_idx)} - if areas is not None and keep_idx: - kept_areas = [float(areas[i]) for i in keep_idx] - slice_summary["total_area"] = float(sum(kept_areas)) - slice_summary["mean_area"] = float(sum(kept_areas) / len(kept_areas)) - if intensity_img is not None: - mean_int = table.get("mean_intensity") - if mean_int is not None and keep_idx: - kept_mean_int = [float(mean_int[i]) for i in keep_idx] - slice_summary["mean_mean_intensity"] = float(sum(kept_mean_int) / len(kept_mean_int)) - per_slice.append(slice_summary) - - # Extract ROIs and attach slice/intensity metadata + payload = [extract_fields(item, getattr(options, "fields", None)) for item in seq] + + if options.wrap_list and isinstance(payload, list): + payload = {"total_items": len(payload), "results": payload} + + return json.dumps(payload, indent=options.indent, default=str) + + +def _single_file_writer( + render: Callable[[Any, Any], str], + *, + validate_payload: Optional[Callable[[Any, Any], None]] = None, +) -> Callable[[Any, Any, "MaterializationContext"], list[Output]]: + def write(data: Any, options: Any, ctx: MaterializationContext) -> list[Output]: + payload = _select_payload(data, options) + if validate_payload is not None: + validate_payload(payload, options) + return [Output(path=_output_path(ctx, options), content=render(payload, options))] + + return write + + +def register_single_file_writer( + options_type: type, + fmt: MaterializationFormat, + *, + render: Callable[[Any, Any], str], + validate_payload: Optional[Callable[[Any, Any], None]] = None, + primary_path: Optional[Callable[[list[Output]], str]] = None, +) -> None: + writer_for(options_type, fmt, primary_path=primary_path)( + _single_file_writer(render, validate_payload=validate_payload) + ) + + +register_single_file_writer(CsvOptions, MaterializationFormat.CSV, render=_render_csv) +register_single_file_writer(JsonOptions, MaterializationFormat.JSON, render=_render_json) + + +def _validate_text(payload: Any, options: TextOptions) -> None: + if not isinstance(payload, str): + raise TypeError(f"TextOptions expects a str payload, got {type(payload).__name__}") + + +register_single_file_writer( + TextOptions, + MaterializationFormat.TEXT, + render=lambda payload, _options: payload, + validate_payload=_validate_text, +) + + +def _roi_primary_path(outs: list[Output]) -> str: + for out in outs: + if out.path.endswith(".roi.zip"): + return out.path + return outs[0].path if outs else "" + + +@writer_for(ROIOptions, MaterializationFormat.ROI_ZIP, primary_path=_roi_primary_path) +def _write_roi_zip(data: Any, options: ROIOptions, ctx: MaterializationContext) -> list[Output]: + from polystore.roi import extract_rois_from_labeled_mask + + data = _select_payload(data, options) + paths = ctx.paths(options) + roi_path = paths.with_suffix(options.roi_suffix) + summary_path = paths.with_suffix(options.summary_suffix) + + if _is_empty(data): + return [Output(path=summary_path, content="No segmentation masks generated (empty data)\n")] + + masks = _as_sequence(data) + + all_rois: list[Any] = [] + for mask in masks: rois = extract_rois_from_labeled_mask( - labels, - min_area=min_area, - extract_contours=extract_contours, + mask, + min_area=options.min_area, + extract_contours=options.extract_contours, ) + all_rois.extend(rois) - intensity_by_label: Dict[int, Dict[str, Any]] = {} - if intensity_img is not None and kept_labels: - for i in keep_idx: - label_id = int(labels_col[i]) - intensity_by_label[label_id] = { - k: _coerce_jsonable(table[k][i]) - for k in intensity_properties - if k in table - } - - for roi in rois: - label_id = int(roi.metadata.get("label", 0)) - metadata = dict(roi.metadata) - metadata["slice_index"] = z_idx - if label_id in intensity_by_label: - metadata.update(intensity_by_label[label_id]) - all_rois.append(ROI(shapes=roi.shapes, metadata=metadata)) - - summary = { - "analysis_type": analysis_type, - "total_slices": len(label_slices), - "total_regions": len(rows), - "regions_per_slice": per_slice, - "details_csv": csv_path, - "roi_zip": roi_path, - "properties": base_properties, - "intensity_properties": intensity_properties if intensity_slices else [], - } - - # JSON summary always (even if empty) - json_content = json.dumps(summary, indent=2, default=str) - for backend in backends: - _prepare_output_path(filemanager, backend, json_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(json_content, json_path, backend, **kwargs) - - # CSV details if any rows - if rows: - df = pd.DataFrame(rows) - csv_content = df.to_csv(index=False) - for backend in backends: - _prepare_output_path(filemanager, backend, csv_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(csv_content, csv_path, backend, **kwargs) - - # ROI zip if any ROIs + outs: list[Output] = [] if all_rois: - for backend in backends: - _prepare_output_path(filemanager, backend, roi_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(all_rois, roi_path, backend, **kwargs) + outs.append(Output(path=roi_path, content=all_rois)) - return json_path + summary = f"Segmentation ROIs: {len(all_rois)} cells\nZ-planes: {len(masks)}\n" + if all_rois: + summary += f"ROI file: {roi_path}\n" + else: + summary += "No ROIs extracted (all regions below min_area threshold)\n" + outs.append(Output(path=summary_path, content=summary)) + return outs -def _get_result_attr(obj: Any, name: str, default: Any = None) -> Any: - if isinstance(obj, dict): - return obj.get(name, default) - return getattr(obj, name, default) +@writer_for(TiffStackOptions, MaterializationFormat.TIFF_STACK) +def _write_tiff_stack(data: Any, options: TiffStackOptions, ctx: MaterializationContext) -> list[Output]: + data = _select_payload(data, options) + paths = ctx.paths(options) + base_name = paths.name + if _is_empty(data): + summary_path = paths.with_suffix(options.summary_suffix) + return [Output(path=summary_path, content=options.empty_summary)] -@register_materializer("cell_counts", requires_arbitrary_files=True) -def _materialize_cell_counts( - data: List[Any], - path: str, - filemanager, - backends: List[str], - backend_kwargs: Dict[str, Dict[str, Any]], - spec: MaterializationSpec, - context: Any, - extra_inputs: Dict[str, Any], -) -> str: - if not data: - logger.warning("CELL_COUNT materializer called with empty data") - return path + if isinstance(data, (list, tuple)): + slices = list(data) + else: + ndim = getattr(data, "ndim", None) + if ndim == 3: + slices = [data[i] for i in range(data.shape[0])] # type: ignore[index] + else: + slices = [data] - options = spec.options - strip_roi = options.get("strip_roi_suffix", False) - base_path = _generate_output_path(path, "", "", strip_roi=strip_roi) - json_path = f"{base_path}.json" - csv_path = f"{base_path}_details.csv" + outs: list[Output] = [] + for i, arr in enumerate(slices): + filename = str(paths.parent / f"{base_name}{options.slice_pattern.format(index=i)}") + out_arr = arr + if options.normalize_uint8 and getattr(out_arr, "dtype", None) != "uint8": + max_val = getattr(out_arr, "max", lambda: 0)() + out_arr = (out_arr * 255).astype("uint8") if max_val <= 1.0 else out_arr.astype("uint8") + outs.append(Output(path=filename, content=out_arr)) + + summary_path = paths.with_suffix(options.summary_suffix) + first = slices[0] if slices else None + summary_content = ( + f"Images saved: {len(slices)} files\n" + f"Base filename pattern: {base_name}{options.slice_pattern}\n" + f"Image dtype: {getattr(first, 'dtype', 'unknown')}\n" + f"Image shape: {getattr(first, 'shape', 'unknown')}\n" + ) + outs.append(Output(path=summary_path, content=summary_content)) + return outs - is_multi_channel = _get_result_attr(data[0], "chan_1_results") is not None - if is_multi_channel: - summary, rows = _build_cell_counts_multi_summary(data) - else: - summary, rows = _build_cell_counts_single_summary(data) - - json_content = json.dumps(summary, indent=2, default=str) - - for backend in backends: - _prepare_output_path(filemanager, backend, json_path) - _prepare_output_path(filemanager, backend, csv_path) - kwargs = backend_kwargs.get(backend, {}) - filemanager.save(json_content, json_path, backend, **kwargs) - if rows: - df = pd.DataFrame(rows) - filemanager.save(df.to_csv(index=False), csv_path, backend, **kwargs) - - return json_path - - -def _build_cell_counts_single_summary(data: List[Any]) -> tuple[Dict[str, Any], List[Dict[str, Any]]]: - summary = { - "analysis_type": "single_channel_cell_counting", - "total_slices": len(data), - "results_per_slice": [] - } - rows: List[Dict[str, Any]] = [] - - total_cells = 0 - for result in data: - slice_index = _get_result_attr(result, "slice_index") - method = _get_result_attr(result, "method") - cell_count = _get_result_attr(result, "cell_count", 0) - cell_positions = _get_result_attr(result, "cell_positions", []) or [] - cell_areas = _get_result_attr(result, "cell_areas", []) or [] - cell_intensities = _get_result_attr(result, "cell_intensities", []) or [] - detection_confidence = _get_result_attr(result, "detection_confidence", []) or [] - parameters_used = _get_result_attr(result, "parameters_used", {}) or {} - - total_cells += cell_count - summary["results_per_slice"].append({ - "slice_index": slice_index, - "method": method, - "cell_count": cell_count, - "avg_cell_area": float(sum(cell_areas) / len(cell_areas)) if cell_areas else 0, - "avg_cell_intensity": float(sum(cell_intensities) / len(cell_intensities)) if cell_intensities else 0, - "parameters": parameters_used - }) - - for i, (pos, area, intensity, confidence) in enumerate(zip( - cell_positions, cell_areas, cell_intensities, detection_confidence - )): - rows.append({ - "slice_index": slice_index, - "cell_id": f"slice_{slice_index}_cell_{i}", - "x_position": pos[0], - "y_position": pos[1], - "cell_area": area, - "cell_intensity": intensity, - "detection_confidence": confidence, - "detection_method": method - }) - - summary["total_cells_all_slices"] = total_cells - summary["average_cells_per_slice"] = total_cells / len(data) if data else 0 - return summary, rows - - -def _build_cell_counts_multi_summary(data: List[Any]) -> tuple[Dict[str, Any], List[Dict[str, Any]]]: - summary = { - "analysis_type": "multi_channel_cell_counting_colocalization", - "total_slices": len(data), - "colocalization_summary": { - "total_chan_1_cells": 0, - "total_chan_2_cells": 0, - "total_colocalized": 0, - "average_colocalization_percentage": 0 - }, - "results_per_slice": [] - } - rows: List[Dict[str, Any]] = [] - - total_coloc_pct = 0 - for result in data: - chan_1 = _get_result_attr(result, "chan_1_results") - chan_2 = _get_result_attr(result, "chan_2_results") - colocalized_count = _get_result_attr(result, "colocalized_count", 0) - colocalization_percentage = _get_result_attr(result, "colocalization_percentage", 0) - chan_1_only = _get_result_attr(result, "chan_1_only_count", 0) - chan_2_only = _get_result_attr(result, "chan_2_only_count", 0) - colocalization_method = _get_result_attr(result, "colocalization_method") - colocalization_metrics = _get_result_attr(result, "colocalization_metrics", {}) or {} - overlap_positions = _get_result_attr(result, "overlap_positions", []) or [] - slice_index = _get_result_attr(result, "slice_index") - - summary["colocalization_summary"]["total_chan_1_cells"] += _get_result_attr(chan_1, "cell_count", 0) - summary["colocalization_summary"]["total_chan_2_cells"] += _get_result_attr(chan_2, "cell_count", 0) - summary["colocalization_summary"]["total_colocalized"] += colocalized_count - total_coloc_pct += colocalization_percentage - - summary["results_per_slice"].append({ - "slice_index": slice_index, - "chan_1_count": _get_result_attr(chan_1, "cell_count", 0), - "chan_2_count": _get_result_attr(chan_2, "cell_count", 0), - "colocalized_count": colocalized_count, - "colocalization_percentage": colocalization_percentage, - "chan_1_only": chan_1_only, - "chan_2_only": chan_2_only, - "colocalization_method": colocalization_method, - "colocalization_metrics": colocalization_metrics - }) - - for i, pos in enumerate(overlap_positions): - rows.append({ - "slice_index": slice_index, - "colocalization_id": f"slice_{slice_index}_coloc_{i}", - "x_position": pos[0], - "y_position": pos[1], - "colocalization_method": colocalization_method - }) - - summary["colocalization_summary"]["average_colocalization_percentage"] = ( - total_coloc_pct / len(data) if data else 0 - ) - return summary, rows - - -# Convenience factory functions (return specs, not callables) - -def csv_materializer( - fields: Optional[List[str]] = None, - filename_suffix: str = ".csv", - analysis_type: Optional[str] = None, - include_metadata: bool = True, - strip_roi_suffix: bool = False -) -> MaterializationSpec: - return MaterializationSpec( - handler="csv", - options={ - "fields": fields, - "filename_suffix": filename_suffix, - "analysis_type": analysis_type, - "include_metadata": include_metadata, - "strip_roi_suffix": strip_roi_suffix - } - ) +@dataclass(frozen=True, init=False) +class MaterializationSpec: + """Declarative materialization spec. + The spec is a list of *writer options* objects. Writer dispatch is inferred + from the option type. + """ -def json_materializer( - fields: Optional[List[str]] = None, - filename_suffix: str = ".json", - analysis_type: Optional[str] = None, - include_metadata: bool = True, - strip_roi_suffix: bool = False -) -> MaterializationSpec: - return MaterializationSpec( - handler="json", - options={ - "fields": fields, - "filename_suffix": filename_suffix, - "analysis_type": analysis_type, - "include_metadata": include_metadata, - "strip_roi_suffix": strip_roi_suffix - } - ) + outputs: Tuple[Any, ...] + allowed_backends: Optional[List[str]] + primary: int + def __init__(self, *outputs: Any, allowed_backends: Optional[List[str]] = None, primary: int = 0): + if len(outputs) == 1 and isinstance(outputs[0], (list, tuple)): + outputs = tuple(outputs[0]) -def dual_materializer( - fields: Optional[List[str]] = None, - summary_fields: Optional[List[str]] = None, - analysis_type: Optional[str] = None, - include_metadata: bool = True, - strip_roi_suffix: bool = False -) -> MaterializationSpec: - return MaterializationSpec( - handler="dual", - options={ - "fields": fields, - "summary_fields": summary_fields, - "analysis_type": analysis_type, - "include_metadata": include_metadata, - "strip_roi_suffix": strip_roi_suffix - } - ) + if not outputs: + raise ValueError("MaterializationSpec requires at least one output options object") + for opt in outputs: + if isinstance(opt, dict): + raise TypeError("dict-based materialization options are not supported") + if type(opt) not in _WRITERS_BY_OPTIONS: + raise ValueError( + f"No writer registered for options type {type(opt).__name__}. " + f"Registered: {[t.__name__ for t in _WRITERS_BY_OPTIONS.keys()]}" + ) -def materializer_spec( - handler: str, - *, - options: Optional[Dict[str, Any]] = None, - allowed_backends: Optional[List[str]] = None -) -> MaterializationSpec: - return MaterializationSpec( - handler=handler, - options=options or {}, - allowed_backends=allowed_backends - ) + if primary < 0 or primary >= len(outputs): + raise IndexError("MaterializationSpec.primary out of range") + object.__setattr__(self, "outputs", tuple(outputs)) + object.__setattr__(self, "allowed_backends", allowed_backends) + object.__setattr__(self, "primary", primary) -def roi_zip_materializer( - *, - min_area: int = 10, - extract_contours: bool = True, - roi_suffix: str = "_rois.roi.zip", - summary_suffix: str = "_segmentation_summary.txt", - strip_roi_suffix: bool = False -) -> MaterializationSpec: - return MaterializationSpec( - handler="roi_zip", - options={ - "min_area": min_area, - "extract_contours": extract_contours, - "roi_suffix": roi_suffix, - "summary_suffix": summary_suffix, - "strip_roi_suffix": strip_roi_suffix - } - ) + @classmethod + def __objectstate_rebuild__( + cls, + *, + outputs: Tuple[Any, ...], + allowed_backends: Optional[List[str]] = None, + primary: int = 0, + ) -> "MaterializationSpec": + # Rebuild via the normal constructor to keep validation behavior. + return cls(*outputs, allowed_backends=allowed_backends, primary=primary) -def regionprops_materializer( - *, - min_area: int = 10, - extract_contours: bool = True, - roi_suffix: str = "_rois.roi.zip", - details_suffix: str = "_details.csv", - json_suffix: str = ".json", - analysis_type: str = "regionprops", - properties: Optional[List[str]] = None, - intensity_properties: Optional[List[str]] = None, - require_intensity: bool = False, - intensity_source: str | None = "step_output", - intensity_group_by: str | None = None, -) -> MaterializationSpec: - inputs: Dict[str, Any] = {} - if intensity_source is not None: - inputs["intensity"] = { - "kind": "image_slices", - "source": intensity_source, - "group_by": intensity_group_by, - } - return MaterializationSpec( - handler="regionprops", - options={ - "min_area": min_area, - "extract_contours": extract_contours, - "roi_suffix": roi_suffix, - "details_suffix": details_suffix, - "json_suffix": json_suffix, - "analysis_type": analysis_type, - "properties": properties, - "intensity_properties": intensity_properties, - "require_intensity": require_intensity, - "inputs": inputs, - }, - ) +def _normalize_backends(backends: Sequence[str] | str) -> list[str]: + if isinstance(backends, str): + return [backends] + return list(backends) + +def _validate_allowed_backends(spec: MaterializationSpec, backends: list[str]) -> None: + if not spec.allowed_backends: + return + invalid = [b for b in backends if b not in spec.allowed_backends] + if invalid: + raise ValueError(f"Backend(s) {invalid} not in allowed backends for this spec: {spec.allowed_backends}") -def tiff_stack_materializer( + +def materialize( + spec: MaterializationSpec, + data: Any, + path: str, + filemanager: Any, + backends: Sequence[str] | str, + backend_kwargs: Optional[Dict[str, Dict[str, Any]]] = None, + context: Any = None, + extra_inputs: Optional[Dict[str, Any]] = None, *, - normalize_uint8: bool = False, - summary_suffix: str = "_summary.txt", - empty_summary: str = "No images generated (empty data)\n", - strip_roi_suffix: bool = False -) -> MaterializationSpec: - return MaterializationSpec( - handler="tiff_stack", - options={ - "normalize_uint8": normalize_uint8, - "summary_suffix": summary_suffix, - "empty_summary": empty_summary, - "strip_roi_suffix": strip_roi_suffix - } + write_mode: WriteMode = WriteMode.OVERWRITE, +) -> str: + """Materialize data to one or more backends.""" + + normalized_backends = _normalize_backends(backends) + _validate_allowed_backends(spec, normalized_backends) + + ctx = MaterializationContext( + base_path=path, + backends=normalized_backends, + backend_kwargs=backend_kwargs or {}, + filemanager=filemanager, + extra_inputs=extra_inputs or {}, + context=context, + write_mode=write_mode, ) + + primary_path = "" + + for i, opt in enumerate(spec.outputs): + writer = _WRITERS_BY_OPTIONS[type(opt)] + outs = writer.write(data, opt, ctx) + for out in outs: + ctx.saver.save(out.content, out.path) + if i == spec.primary: + primary_path = writer.primary_path(outs) + + return primary_path diff --git a/openhcs/processing/materialization/options.py b/openhcs/processing/materialization/options.py new file mode 100644 index 000000000..b13002571 --- /dev/null +++ b/openhcs/processing/materialization/options.py @@ -0,0 +1,84 @@ +"""Typed writer options. + +Greenfield design: +- the abstraction boundary is the *output format* (CSV/JSON/ROI_ZIP/TIFF/etc) +- options types are used for dispatch (metaprogramming-friendly) +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional + + +@dataclass(frozen=True) +class FileOutputOptions: + """Common filename + strip behavior.""" + + filename_suffix: str = "" + strip_roi_suffix: bool = False + strip_pkl: bool = True + + +@dataclass(frozen=True) +class SourceOptions: + """Select a sub-value from the input before writing. + + The selector is a dot-path. For dicts, keys are used. For objects, attributes. + Example: source="branch_data". + """ + + source: Optional[str] = None + + +@dataclass(frozen=True) +class TabularExtractionOptions: + """Generic extraction options for tabular writers.""" + + fields: Optional[List[str]] = None + row_field: Optional[str] = None + row_columns: Dict[str, str] = field(default_factory=dict) + row_unpacker: Optional[Callable[[Any], List[Dict[str, Any]]]] = None + + +@dataclass(frozen=True) +class CsvOptions(FileOutputOptions, SourceOptions, TabularExtractionOptions): + """CSV writer options.""" + + filename_suffix: str = "_details.csv" + + +@dataclass(frozen=True) +class JsonOptions(FileOutputOptions, SourceOptions, TabularExtractionOptions): + """JSON writer options.""" + + filename_suffix: str = ".json" + indent: int = 2 + wrap_list: bool = False + + +@dataclass(frozen=True) +class ROIOptions(FileOutputOptions, SourceOptions): + """ROI ZIP writer options.""" + + min_area: int = 10 + extract_contours: bool = True + roi_suffix: str = "_rois.roi.zip" + summary_suffix: str = "_segmentation_summary.txt" + + +@dataclass(frozen=True) +class TiffStackOptions(FileOutputOptions, SourceOptions): + """TIFF stack writer options (per-slice TIFF + summary).""" + + normalize_uint8: bool = False + slice_pattern: str = "_slice_{index:03d}.tif" + summary_suffix: str = "_summary.txt" + empty_summary: str = "No images generated (empty data)\n" + + +@dataclass(frozen=True) +class TextOptions(FileOutputOptions, SourceOptions): + """Text writer options.""" + + filename_suffix: str = ".txt" diff --git a/openhcs/processing/materialization/presets.py b/openhcs/processing/materialization/presets.py new file mode 100644 index 000000000..1b07c2e72 --- /dev/null +++ b/openhcs/processing/materialization/presets.py @@ -0,0 +1,142 @@ +"""Convenience presets for common materialization patterns. + +These are intentionally small: they make analysis modules read declaratively +without repeating JsonOptions/CsvOptions boilerplate. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, List, Optional + +from openhcs.processing.materialization.core import MaterializationSpec +from openhcs.processing.materialization.options import ( + CsvOptions, + JsonOptions, + ROIOptions, + TextOptions, + TiffStackOptions, +) + + +def json_only( + *, + source: Optional[str] = None, + suffix: str = ".json", + indent: int = 2, + wrap_list: bool = False, + allowed_backends: Optional[List[str]] = None, +) -> MaterializationSpec: + return MaterializationSpec( + JsonOptions(source=source, filename_suffix=suffix, indent=indent, wrap_list=wrap_list), + allowed_backends=allowed_backends, + ) + + +def csv_only( + *, + source: Optional[str] = None, + suffix: str = "_details.csv", + fields: Optional[List[str]] = None, + row_field: Optional[str] = None, + row_columns: Optional[Dict[str, str]] = None, + row_unpacker: Optional[Callable[[Any], List[Dict[str, Any]]]] = None, + allowed_backends: Optional[List[str]] = None, +) -> MaterializationSpec: + return MaterializationSpec( + CsvOptions( + source=source, + filename_suffix=suffix, + fields=fields, + row_field=row_field, + row_columns=row_columns or {}, + row_unpacker=row_unpacker, + ), + allowed_backends=allowed_backends, + ) + + +def json_and_csv( + *, + json_source: Optional[str] = None, + csv_source: Optional[str] = None, + json_suffix: str = ".json", + csv_suffix: str = "_details.csv", + json_indent: int = 2, + wrap_list: bool = False, + fields: Optional[List[str]] = None, + row_field: Optional[str] = None, + row_columns: Optional[Dict[str, str]] = None, + row_unpacker: Optional[Callable[[Any], List[Dict[str, Any]]]] = None, + primary: str = "json", + allowed_backends: Optional[List[str]] = None, +) -> MaterializationSpec: + primary_idx = 0 if primary == "json" else 1 + return MaterializationSpec( + JsonOptions( + source=json_source, + filename_suffix=json_suffix, + indent=json_indent, + wrap_list=wrap_list, + ), + CsvOptions( + source=csv_source, + filename_suffix=csv_suffix, + fields=fields, + row_field=row_field, + row_columns=row_columns or {}, + row_unpacker=row_unpacker, + ), + primary=primary_idx, + allowed_backends=allowed_backends, + ) + + +def roi_zip( + *, + source: Optional[str] = None, + min_area: int = 10, + extract_contours: bool = True, + roi_suffix: str = "_rois.roi.zip", + summary_suffix: str = "_segmentation_summary.txt", + allowed_backends: Optional[List[str]] = None, +) -> MaterializationSpec: + return MaterializationSpec( + ROIOptions( + source=source, + min_area=min_area, + extract_contours=extract_contours, + roi_suffix=roi_suffix, + summary_suffix=summary_suffix, + ), + allowed_backends=allowed_backends, + ) + + +def tiff_stack( + *, + source: Optional[str] = None, + normalize_uint8: bool = False, + slice_pattern: str = "_slice_{index:03d}.tif", + summary_suffix: str = "_summary.txt", + empty_summary: str = "No images generated (empty data)\n", + allowed_backends: Optional[List[str]] = None, +) -> MaterializationSpec: + return MaterializationSpec( + TiffStackOptions( + source=source, + normalize_uint8=normalize_uint8, + slice_pattern=slice_pattern, + summary_suffix=summary_suffix, + empty_summary=empty_summary, + ), + allowed_backends=allowed_backends, + ) + + +def text_only( + *, + source: Optional[str] = None, + suffix: str = ".txt", + allowed_backends: Optional[List[str]] = None, +) -> MaterializationSpec: + return MaterializationSpec(TextOptions(source=source, filename_suffix=suffix), allowed_backends=allowed_backends) diff --git a/openhcs/processing/materialization/utils.py b/openhcs/processing/materialization/utils.py new file mode 100644 index 000000000..1f5cf82b2 --- /dev/null +++ b/openhcs/processing/materialization/utils.py @@ -0,0 +1,119 @@ +"""Shared helpers for materialization handlers. + +Keep this module free of backend/registry concerns. +""" + +from __future__ import annotations + +from dataclasses import fields, is_dataclass +from typing import Any, Dict, List, Optional, Tuple + + +def extract_fields(item: Any, field_names: Optional[List[str]] = None) -> Dict[str, Any]: + """Extract fields from dataclass, dict, or pandas objects. + + Supports: + - dataclass instances (uses dataclass reflection) + - dicts (uses dict keys) + - pandas DataFrames (uses column names) + - pandas Series (uses index) + """ + + # pandas (optional) + try: + import pandas as pd + + if isinstance(item, pd.DataFrame): + if field_names: + return {f: item[f].tolist() for f in field_names if f in item.columns} + return {col: item[col].tolist() for col in item.columns} + + if isinstance(item, pd.Series): + if field_names: + return {f: item[f] for f in field_names if f in item.index} + return item.to_dict() + except ImportError: + pass + + if is_dataclass(item): + if field_names: + return {f: getattr(item, f, None) for f in field_names if hasattr(item, f)} + return {f.name: getattr(item, f.name) for f in fields(item)} + + if isinstance(item, dict): + if field_names: + return {f: item.get(f) for f in field_names if f in item} + return item + + return {"value": item} + + +def coerce_jsonable(value: Any) -> Any: + """Convert numpy scalars/arrays to JSON-serializable Python types.""" + try: + import numpy as np + + if isinstance(value, np.generic): + return value.item() + if isinstance(value, np.ndarray): + return value.tolist() + except Exception: + pass + return value + + +def normalize_slices(obj: Any, *, name: str): + """Normalize input to list of 2D numpy arrays.""" + import numpy as np + + if obj is None: + return [] + if isinstance(obj, list): + return [np.asarray(x) for x in obj] + arr = np.asarray(obj) + if arr.ndim == 2: + return [arr] + if arr.ndim == 3: + return [arr[i] for i in range(arr.shape[0])] + raise ValueError(f"{name} must be a 2D/3D array or list of 2D arrays, got shape {arr.shape}") + + +def discover_array_fields(item: Any) -> List[str]: + """Discover array/tuple fields in a dataclass instance.""" + if not hasattr(item, "__dataclass_fields__"): + return [] + + from typing import get_origin + + array_fields: List[str] = [] + for f in fields(item): + value = getattr(item, f.name, None) + origin = hasattr(f.type, "__origin__") and get_origin(f.type) + if origin in (list, List, tuple, Tuple): + array_fields.append(f.name) + elif isinstance(value, list) and value and isinstance(value[0], (tuple, list)): + array_fields.append(f.name) + return array_fields + + +def expand_array_field( + array_data: List[Any], + base_row: Dict[str, Any], + row_columns: Dict[str, str], +) -> List[Dict[str, Any]]: + """Expand an array field into multiple rows.""" + if not array_data: + return [base_row] + + rows: List[Dict[str, Any]] = [] + for elem in array_data: + if isinstance(elem, (list, tuple)): + cols = { + col: elem[int(idx)] + for idx, col in row_columns.items() + if str(idx).isdigit() and int(idx) < len(elem) + } + else: + cols = {} + rows.append({**base_row, **cols}) + return rows diff --git a/openhcs/pyqt_gui/__init__.py b/openhcs/pyqt_gui/__init__.py index 2e5630213..9edffd088 100644 --- a/openhcs/pyqt_gui/__init__.py +++ b/openhcs/pyqt_gui/__init__.py @@ -21,8 +21,35 @@ __version__ = "1.0.0" __author__ = "OpenHCS Development Team" -from openhcs.pyqt_gui.main import OpenHCSMainWindow -from openhcs.pyqt_gui.app import OpenHCSPyQtApp +# Lazy-load GUI classes to support environment-specific Qt platform initialization. +# The OpenHCS GUI requires platform-specific Qt configuration (WSL2 needs Wayland, +# macOS needs Cocoa plugin path, etc.) before PyQt6 is imported. This configuration +# happens in launch.py:setup_qt_platform(). Lazy loading via __getattr__ ensures that +# GUI imports are deferred until after platform setup completes. + +def __getattr__(name: str): + """Lazy import GUI classes to defer PyQt6 initialization. + + Defers importing OpenHCSMainWindow and OpenHCSPyQtApp until accessed, + allowing launch.py:setup_qt_platform() to configure Qt before any + PyQt6 libraries are loaded. + + Args: + name: The attribute being accessed. + + Returns: + The requested class. + + Raises: + AttributeError: If the requested attribute doesn't exist. + """ + if name == "OpenHCSMainWindow": + from openhcs.pyqt_gui.main import OpenHCSMainWindow + return OpenHCSMainWindow + elif name == "OpenHCSPyQtApp": + from openhcs.pyqt_gui.app import OpenHCSPyQtApp + return OpenHCSPyQtApp + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") __all__ = [ "OpenHCSMainWindow", diff --git a/openhcs/pyqt_gui/app.py b/openhcs/pyqt_gui/app.py index 245e32f65..5d17ae9c2 100644 --- a/openhcs/pyqt_gui/app.py +++ b/openhcs/pyqt_gui/app.py @@ -11,12 +11,15 @@ from pathlib import Path from PyQt6.QtWidgets import QApplication, QMessageBox +from PyQt6.QtCore import qInstallMessageHandler from PyQt6.QtGui import QIcon from openhcs.core.config import GlobalPipelineConfig from polystore.base import storage_registry from polystore.filemanager import FileManager +from objectstate import spawn_thread_with_context +from pyqt_reactive.utils.scroll_filter import install_shift_wheel_scrolling from openhcs.pyqt_gui.main import OpenHCSMainWindow logger = logging.getLogger(__name__) @@ -25,53 +28,67 @@ class OpenHCSPyQtApp(QApplication): """ OpenHCS PyQt6 Application. - + Main application class that manages global state, configuration, and the main window lifecycle. """ - - def __init__(self, argv: list, global_config: Optional[GlobalPipelineConfig] = None): + + def __init__( + self, argv: list, global_config: Optional[GlobalPipelineConfig] = None + ): """ Initialize the OpenHCS PyQt6 application. - + Args: argv: Command line arguments global_config: Global configuration (uses default if None) """ super().__init__(argv) - + + def _qt_message_handler(msg_type, context, message): + if "QTextCursor::setPosition" in message: + logger.warning("Qt: %s", message) + + qInstallMessageHandler(_qt_message_handler) + # Application metadata self.setApplicationName("OpenHCS") self.setApplicationVersion("1.0.0") self.setOrganizationName("OpenHCS Development Team") self.setOrganizationDomain("openhcs.org") - + # Global configuration self.global_config = global_config or GlobalPipelineConfig() - + # Shared components self.storage_registry = storage_registry self.file_manager = FileManager(self.storage_registry) - + # Main window self.main_window: Optional[OpenHCSMainWindow] = None - + # Setup application self.setup_application() - + + # Install global Shift+Wheel horizontal scrolling + self._scroll_filter = install_shift_wheel_scrolling(self) + logger.debug("Installed global Shift+Wheel horizontal scrolling") + logger.info("OpenHCS PyQt6 application initialized") - + def setup_application(self): """Setup application-wide configuration.""" + # Start async storage registry initialization in background thread - import threading def init_storage_registry_background(): from polystore.base import ensure_storage_registry + ensure_storage_registry() logger.info("Storage registry initialized in background") - thread = threading.Thread(target=init_storage_registry_background, daemon=True, name="storage-registry-init") - thread.start() + spawn_thread_with_context( + init_storage_registry_background, name="storage-registry-init" + ) logger.info("Storage registry initialization started in background") # Start async function registry initialization in background thread @@ -79,18 +96,25 @@ def init_storage_registry_background(): # Custom functions are automatically loaded as part of initialize_registry() def init_function_registry_background(): from openhcs.processing.func_registry import initialize_registry + initialize_registry() - logger.info("Function registry initialized in background - virtual modules created") + logger.info( + "Function registry initialized in background - virtual modules created" + ) - func_thread = threading.Thread(target=init_function_registry_background, daemon=True, name="function-registry-init") - func_thread.start() + spawn_thread_with_context( + init_function_registry_background, name="function-registry-init" + ) logger.info("Function registry initialization started in background") # CRITICAL FIX: Establish global config context for lazy dataclass resolution # This was missing and caused placeholder resolution to fall back to static defaults from openhcs.config_framework.global_config import set_global_config_for_editing from openhcs.config_framework.lazy_factory import ensure_global_config_context - from openhcs.config_framework.object_state import ObjectState, ObjectStateRegistry + from openhcs.config_framework.object_state import ( + ObjectState, + ObjectStateRegistry, + ) from openhcs.core.config import GlobalPipelineConfig # Set for editing (UI placeholders) - this uses threading.local() storage @@ -106,7 +130,7 @@ def init_function_registry_background(): object_instance=self.global_config, scope_id="", # Empty string = global scope ) - ObjectStateRegistry.register(global_state) + ObjectStateRegistry.register(global_state, _skip_snapshot=True) # ARCHITECTURAL FIX: Do NOT set contextvars at app startup # contextvars is ONLY for temporary nested contexts (inside with config_context() blocks) @@ -114,10 +138,15 @@ def init_function_registry_background(): # Placeholder resolution will automatically fall back to threading.local() via get_base_global_config() # This eliminates the dual storage architecture smell - logger.info("Global configuration context established for lazy dataclass resolution") + logger.info( + "Global configuration context established for lazy dataclass resolution" + ) # Register pyqt-reactor providers (LLM, codegen, log discovery, window factory, etc.) - from openhcs.pyqt_gui.services.reactor_providers import register_reactor_providers + from openhcs.pyqt_gui.services.reactor_providers import ( + register_reactor_providers, + ) + register_reactor_providers() # Set application icon (if available) @@ -127,22 +156,22 @@ def init_function_registry_background(): # Setup exception handling sys.excepthook = self.handle_exception - + def create_main_window(self) -> OpenHCSMainWindow: """ Create and show the main window. - + Returns: Created main window """ if self.main_window is None: self.main_window = OpenHCSMainWindow(self.global_config) - + # Connect application-level signals self.main_window.config_changed.connect(self.on_config_changed) - + return self.main_window - + def show_main_window(self): """Show the main window.""" if self.main_window is None: @@ -155,22 +184,23 @@ def show_main_window(self): # Trigger deferred initialization AFTER window is visible # This includes log viewer and default windows (pipeline editor) from PyQt6.QtCore import QTimer + QTimer.singleShot(100, self.main_window._deferred_initialization) - + def on_config_changed(self, new_config: GlobalPipelineConfig): """ Handle global configuration changes. - + Args: new_config: New global configuration """ self.global_config = new_config logger.info("Global configuration updated") - + def handle_exception(self, exc_type, exc_value, exc_traceback): """ Handle uncaught exceptions. - + Args: exc_type: Exception type exc_value: Exception value @@ -180,26 +210,23 @@ def handle_exception(self, exc_type, exc_value, exc_traceback): # Handle Ctrl+C gracefully sys.__excepthook__(exc_type, exc_value, exc_traceback) return - + # Log the exception logger.critical( - "Uncaught exception", - exc_info=(exc_type, exc_value, exc_traceback) + "Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback) ) - + # Show error dialog error_msg = f"An unexpected error occurred:\n\n{exc_type.__name__}: {exc_value}" - + if self.main_window: - QMessageBox.critical( - self.main_window, - "Unexpected Error", - error_msg - ) + QMessageBox.critical(self.main_window, "Unexpected Error", error_msg) else: # No main window - application is in invalid state - raise RuntimeError("Uncaught exception occurred but no main window available for error dialog") - + raise RuntimeError( + "Uncaught exception occurred but no main window available for error dialog" + ) + def run(self) -> int: """ Run the application. @@ -233,7 +260,7 @@ def cleanup(self): self.processEvents() # Clean up main window - if hasattr(self, 'main_window') and self.main_window: + if hasattr(self, "main_window") and self.main_window: # Force close if not already closed if not self.main_window.isHidden(): self.main_window.close() @@ -245,6 +272,7 @@ def cleanup(self): # Force garbage collection import gc + gc.collect() logger.info("Application cleanup completed") @@ -255,5 +283,7 @@ def cleanup(self): if __name__ == "__main__": # Don't run directly - use launch.py instead - print("Use 'python -m openhcs.pyqt_gui' or 'python -m openhcs.pyqt_gui.launch' to start the GUI") + print( + "Use 'python -m openhcs.pyqt_gui' or 'python -m openhcs.pyqt_gui.launch' to start the GUI" + ) sys.exit(1) diff --git a/openhcs/pyqt_gui/config.py b/openhcs/pyqt_gui/config.py index 345495082..ace76ff01 100644 --- a/openhcs/pyqt_gui/config.py +++ b/openhcs/pyqt_gui/config.py @@ -18,9 +18,11 @@ # Declarative Keyboard Shortcuts System # ============================================================================ + @dataclass(frozen=True) class Shortcut: """Single keyboard shortcut binding.""" + key: str # e.g., "Ctrl+Z", "F1", "Ctrl+Shift+S" action: str # Method/action name to invoke description: str # Human-readable description @@ -36,18 +38,36 @@ class ShortcutConfig: """ # Time travel (global - works everywhere) - time_travel_back: Shortcut = Shortcut("Ctrl+Z", "time_travel_back", "Step back in history") - time_travel_forward: Shortcut = Shortcut("Ctrl+Y", "time_travel_forward", "Step forward in history") - time_travel_to_head: Shortcut = Shortcut("Ctrl+Shift+Y", "time_travel_to_head", "Return to present") + time_travel_back: Shortcut = Shortcut( + "Ctrl+Z", "time_travel_back", "Step back in history" + ) + time_travel_forward: Shortcut = Shortcut( + "Ctrl+Y", "time_travel_forward", "Step forward in history" + ) + time_travel_to_head: Shortcut = Shortcut( + "Ctrl+Shift+Y", "time_travel_to_head", "Return to present" + ) # Window management - show_plate_manager: Shortcut = Shortcut("Ctrl+P", "show_plate_manager", "Show Plate Manager") - show_pipeline_editor: Shortcut = Shortcut("Ctrl+E", "show_pipeline_editor", "Show Pipeline Editor") - show_image_browser: Shortcut = Shortcut("Ctrl+I", "show_image_browser", "Show Image Browser") + show_plate_manager: Shortcut = Shortcut( + "Ctrl+P", "show_plate_manager", "Show Plate Manager" + ) + show_pipeline_editor: Shortcut = Shortcut( + "Ctrl+E", "show_pipeline_editor", "Show Pipeline Editor" + ) + show_image_browser: Shortcut = Shortcut( + "Ctrl+I", "show_image_browser", "Show Image Browser" + ) show_log_viewer: Shortcut = Shortcut("Ctrl+L", "show_log_viewer", "Show Log Viewer") - show_zmq_server_manager: Shortcut = Shortcut("Ctrl+M", "show_zmq_server_manager", "Show ZMQ Server Manager") - show_configuration: Shortcut = Shortcut("Ctrl+G", "show_configuration", "Show Global Configuration") - show_synthetic_plate_generator: Shortcut = Shortcut("Ctrl+Shift+G", "show_synthetic_plate_generator", "Generate Synthetic Plate") + show_zmq_server_manager: Shortcut = Shortcut( + "Ctrl+M", "show_zmq_server_manager", "Show ZMQ Server Manager" + ) + show_configuration: Shortcut = Shortcut( + "Ctrl+G", "show_configuration", "Show Global Configuration" + ) + show_synthetic_plate_generator: Shortcut = Shortcut( + "Ctrl+Shift+G", "show_synthetic_plate_generator", "Generate Synthetic Plate" + ) # Help show_help: Shortcut = Shortcut("F1", "show_help", "Show Documentation") @@ -70,6 +90,7 @@ def get_shortcut_config() -> ShortcutConfig: class PlotTheme(Enum): """Available plot themes for PyQtGraph components.""" + DARK = "dark" LIGHT = "light" AUTO = "auto" # Follow system theme @@ -77,83 +98,91 @@ class PlotTheme(Enum): class UpdateStrategy(Enum): """Update strategies for real-time monitoring components.""" + FIXED_RATE = "fixed_rate" # Fixed FPS regardless of data availability - ADAPTIVE = "adaptive" # Adapt rate based on data changes - ON_DEMAND = "on_demand" # Update only when explicitly requested + ADAPTIVE = "adaptive" # Adapt rate based on data changes + ON_DEMAND = "on_demand" # Update only when explicitly requested @dataclass(frozen=True) class PerformanceMonitorConfig: """Configuration for the system performance monitor widget.""" - + # Update frequency settings update_fps: float = 5.0 """Update frequency in frames per second (FPS). Default: 5 FPS for good performance.""" - + + render_fps: float = 60.0 + """Graph render FPS for smooth interpolation (data collection stays at update_fps).""" + history_duration_seconds: float = 60.0 """Duration of historical data to display in seconds. Default: 60 seconds.""" - + # Display settings plot_theme: PlotTheme = PlotTheme.DARK """Theme for plots and charts.""" - + show_grid: bool = True """Whether to show grid lines on plots.""" - + antialiasing: bool = True """Enable antialiasing for smoother plot rendering.""" - + # Performance settings update_strategy: UpdateStrategy = UpdateStrategy.FIXED_RATE """Strategy for updating the display.""" - + max_data_points: Optional[int] = None """Maximum number of data points to keep. If None, calculated from update_fps and history_duration.""" - + # GPU monitoring settings enable_gpu_monitoring: bool = True """Enable GPU usage monitoring if available.""" - + gpu_temperature_monitoring: bool = True """Enable GPU temperature monitoring if available.""" - + # CPU monitoring settings cpu_frequency_monitoring: bool = True """Enable CPU frequency monitoring.""" - + per_core_cpu_monitoring: bool = False """Monitor individual CPU cores (more detailed but higher overhead).""" - + # Memory monitoring settings detailed_memory_info: bool = True """Include detailed memory information (available, cached, etc.).""" - + # Chart appearance line_width: float = 2.0 """Width of plot lines in pixels.""" - - chart_colors: Dict[str, str] = field(default_factory=lambda: { - 'cpu': 'cyan', - 'ram': 'lime', - 'gpu': 'orange', - 'vram': 'magenta' - }) + + chart_colors: Dict[str, str] = field( + default_factory=lambda: { + "cpu": "cyan", + "ram": "lime", + "gpu": "orange", + "vram": "magenta", + } + ) """Color scheme for different metrics.""" - + def __post_init__(self): """Validate configuration after initialization.""" if self.update_fps <= 0: raise ValueError("update_fps must be positive") + if self.render_fps <= 0: + raise ValueError("render_fps must be positive") if self.history_duration_seconds <= 0: raise ValueError("history_duration_seconds must be positive") if self.line_width <= 0: raise ValueError("line_width must be positive") - + @property def update_interval_seconds(self) -> float: """Calculate update interval in seconds from FPS.""" return 1.0 / self.update_fps - + @property def calculated_max_data_points(self) -> int: """Calculate maximum data points based on FPS and history duration.""" @@ -165,27 +194,27 @@ def calculated_max_data_points(self) -> int: @dataclass(frozen=True) class WindowConfig: """Configuration for main window behavior.""" - + # Window properties default_width: int = 1200 """Default window width in pixels.""" - + default_height: int = 800 """Default window height in pixels.""" - + remember_window_state: bool = True """Remember window size and position between sessions.""" - + floating_by_default: bool = True """Whether main window should be floating (non-tiled) by default.""" - + # Behavior settings confirm_close: bool = True """Show confirmation dialog when closing the application.""" - + minimize_to_tray: bool = False """Minimize to system tray instead of taskbar.""" - + auto_save_interval_minutes: Optional[int] = 5 """Auto-save interval in minutes. None to disable auto-save.""" @@ -193,29 +222,29 @@ class WindowConfig: @dataclass(frozen=True) class StyleConfig: """Configuration for GUI styling and appearance.""" - + # Theme settings theme: PlotTheme = PlotTheme.DARK """Overall application theme.""" - + # Font settings default_font_family: str = "Arial" """Default font family for the application.""" - + default_font_size: int = 10 """Default font size in points.""" - + monospace_font_family: str = "Consolas" """Font family for monospace text (logs, code, etc.).""" - + # Color customization custom_colors: Dict[str, str] = field(default_factory=dict) """Custom color overrides for theme colors.""" - + # Animation settings enable_animations: bool = True """Enable UI animations and transitions.""" - + animation_duration_ms: int = 200 """Duration of animations in milliseconds.""" @@ -223,57 +252,80 @@ class StyleConfig: @dataclass(frozen=True) class LoggingConfig: """Configuration for GUI logging and debugging.""" - + # Log display settings max_log_entries: int = 1000 """Maximum number of log entries to keep in memory.""" - + auto_scroll_logs: bool = True """Automatically scroll to newest log entries.""" - + log_level_filter: str = "INFO" """Minimum log level to display in GUI.""" - + # Log file settings enable_file_logging: bool = True """Enable logging to file.""" - + log_file_max_size_mb: int = 10 """Maximum log file size in MB before rotation.""" - + log_file_backup_count: int = 5 """Number of backup log files to keep.""" +@dataclass(frozen=True) +class ProgressUIConfig: + """Configuration for progress UI update coalescing.""" + + update_fps: float = 30.0 + """Maximum progress UI update rate in frames per second. + + Background threads set a dirty flag on each progress message; + a QTimer fires at this rate and performs the actual UI update. + Higher values give smoother progress display but use more CPU. + """ + + @property + def update_interval_ms(self) -> int: + """Timer interval in milliseconds derived from update_fps.""" + return max(1, int(1000.0 / self.update_fps)) + + @dataclass(frozen=True) class PyQtGUIConfig: """ Root configuration object for the PyQt GUI application. - + This follows the same pattern as GlobalPipelineConfig, providing a centralized, immutable configuration for all GUI components. """ - + # Component configurations - performance_monitor: PerformanceMonitorConfig = field(default_factory=PerformanceMonitorConfig) + performance_monitor: PerformanceMonitorConfig = field( + default_factory=PerformanceMonitorConfig + ) """Configuration for the system performance monitor.""" - + + progress: ProgressUIConfig = field(default_factory=ProgressUIConfig) + """Configuration for progress UI update coalescing.""" + window: WindowConfig = field(default_factory=WindowConfig) """Configuration for main window behavior.""" - + style: StyleConfig = field(default_factory=StyleConfig) """Configuration for GUI styling and appearance.""" - + logging: LoggingConfig = field(default_factory=LoggingConfig) """Configuration for GUI logging.""" - + # Global GUI settings enable_debug_mode: bool = False """Enable debug mode with additional logging and diagnostics.""" - + check_for_updates: bool = True """Check for application updates on startup.""" - + # Future extension points plugin_settings: Dict[str, Any] = field(default_factory=dict) """Settings for GUI plugins and extensions.""" @@ -285,35 +337,30 @@ class PyQtGUIConfig: update_fps=5.0, # 5 FPS for good performance balance history_duration_seconds=60.0, plot_theme=PlotTheme.DARK, - enable_gpu_monitoring=True + enable_gpu_monitoring=True, ) _DEFAULT_WINDOW_CONFIG = WindowConfig( default_width=1200, default_height=800, floating_by_default=True, # User preference for tiling window manager - remember_window_state=True + remember_window_state=True, ) -_DEFAULT_STYLE_CONFIG = StyleConfig( - theme=PlotTheme.DARK, - enable_animations=True -) +_DEFAULT_STYLE_CONFIG = StyleConfig(theme=PlotTheme.DARK, enable_animations=True) _DEFAULT_LOGGING_CONFIG = LoggingConfig( - max_log_entries=1000, - auto_scroll_logs=True, - log_level_filter="INFO" + max_log_entries=1000, auto_scroll_logs=True, log_level_filter="INFO" ) def get_default_pyqt_gui_config() -> PyQtGUIConfig: """ Provides a default instance of PyQtGUIConfig. - + This function provides sensible defaults for the PyQt GUI application, following the same pattern as GlobalPipelineConfig(). - + Returns: PyQtGUIConfig: Default configuration instance """ @@ -324,14 +371,14 @@ def get_default_pyqt_gui_config() -> PyQtGUIConfig: style=_DEFAULT_STYLE_CONFIG, logging=_DEFAULT_LOGGING_CONFIG, enable_debug_mode=False, - check_for_updates=True + check_for_updates=True, ) def create_high_performance_config() -> PyQtGUIConfig: """ Create a high-performance configuration preset. - + Returns: PyQtGUIConfig: High-performance configuration """ @@ -341,18 +388,18 @@ def create_high_performance_config() -> PyQtGUIConfig: history_duration_seconds=30.0, # Shorter history for performance antialiasing=False, # Disable for performance per_core_cpu_monitoring=True, # More detailed monitoring - detailed_memory_info=True + detailed_memory_info=True, ), style=StyleConfig( enable_animations=False # Disable animations for performance - ) + ), ) def create_low_resource_config() -> PyQtGUIConfig: """ Create a low-resource configuration preset. - + Returns: PyQtGUIConfig: Low-resource configuration """ @@ -364,13 +411,14 @@ def create_low_resource_config() -> PyQtGUIConfig: enable_gpu_monitoring=False, # Disable GPU monitoring gpu_temperature_monitoring=False, cpu_frequency_monitoring=False, - detailed_memory_info=False + detailed_memory_info=False, + ), + progress=ProgressUIConfig( + update_fps=10.0, # Lower progress update rate to save CPU ), logging=LoggingConfig( max_log_entries=100, # Fewer log entries - enable_file_logging=False + enable_file_logging=False, ), - style=StyleConfig( - enable_animations=False - ) + style=StyleConfig(enable_animations=False), ) diff --git a/openhcs/pyqt_gui/dialogs/custom_function_manager_dialog.py b/openhcs/pyqt_gui/dialogs/custom_function_manager_dialog.py index 911e47c83..f8d4aab60 100644 --- a/openhcs/pyqt_gui/dialogs/custom_function_manager_dialog.py +++ b/openhcs/pyqt_gui/dialogs/custom_function_manager_dialog.py @@ -19,7 +19,7 @@ from pyqt_reactive.theming import ColorScheme from pyqt_reactive.theming import StyleSheetGenerator from pyqt_reactive.widgets.editors.simple_code_editor import QScintillaCodeEditorDialog -from openhcs.pyqt_gui.windows.base_form_dialog import BaseFormDialog +from pyqt_reactive.widgets.shared import BaseFormDialog from openhcs.processing.custom_functions import CustomFunctionManager from openhcs.processing.custom_functions.signals import custom_function_signals from openhcs.processing.custom_functions.templates import get_default_template diff --git a/openhcs/pyqt_gui/dialogs/function_selector_dialog.py b/openhcs/pyqt_gui/dialogs/function_selector_dialog.py index 3a7bca232..4f654af14 100644 --- a/openhcs/pyqt_gui/dialogs/function_selector_dialog.py +++ b/openhcs/pyqt_gui/dialogs/function_selector_dialog.py @@ -9,8 +9,16 @@ from typing import Callable, Optional, Dict, Any from PyQt6.QtWidgets import ( - QDialog, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, - QTreeWidget, QTreeWidgetItem, QSplitter, QWidget, QSizePolicy + QDialog, + QVBoxLayout, + QHBoxLayout, + QPushButton, + QLabel, + QTreeWidget, + QTreeWidgetItem, + QSplitter, + QWidget, + QSizePolicy, ) from PyQt6.QtCore import Qt, pyqtSignal from PyQt6.QtGui import QFont @@ -50,7 +58,7 @@ def get_function_library(cls, func_name: str, module_path: str) -> str: if registry_name.lower() in module_path.lower(): return display_name - return 'Unknown' + return "Unknown" @classmethod def _build_registry_ownership_cache(cls): @@ -77,16 +85,16 @@ def _extract_library_name(cls, metadata) -> str: # Fallback to module path analysis module_path = metadata.module.lower() - if 'openhcs' in module_path: - return 'OpenHCS' - elif 'cupy' in module_path: - return 'CuPy' - elif 'pyclesperanto' in module_path or 'cle' in module_path: - return 'Pyclesperanto' - elif 'skimage' in module_path: - return 'scikit-image' + if "openhcs" in module_path: + return "OpenHCS" + elif "cupy" in module_path: + return "CuPy" + elif "pyclesperanto" in module_path or "cle" in module_path: + return "Pyclesperanto" + elif "skimage" in module_path: + return "scikit-image" - return 'Unknown' + return "Unknown" class FunctionSelectorDialog(QDialog): @@ -113,8 +121,8 @@ def clear_cache(cls): MIN_HEIGHT = 500 MODULE_COLUMN_WIDTH = 250 DESCRIPTION_COLUMN_WIDTH = 200 - TREE_PROPORTION = 300 # Reduced from 400 to take up less width - TABLE_PROPORTION = 700 # Increased from 600 to give more space to table + TREE_PROPORTION = 180 # Reduced to give more space to function table + TABLE_PROPORTION = 820 # Increased for better function table visibility # Signals function_selected = pyqtSignal(object) # Selected function @@ -150,7 +158,9 @@ def __init__(self, current_function: Optional[Callable] = None, parent=None): # Connect to custom function signals for auto-refresh custom_function_signals.functions_changed.connect(self._on_functions_changed) - logger.debug(f"Function selector initialized with {len(self.all_functions_metadata)} functions") + logger.debug( + f"Function selector initialized with {len(self.all_functions_metadata)} functions" + ) def _load_function_data(self) -> None: """Load ALL functions from registries (not just FUNC_REGISTRY subset).""" @@ -173,30 +183,32 @@ def _load_function_data(self) -> None: # Handle composite keys (backend:function_name) from registry service for composite_key, metadata in unified_functions.items(): # Extract backend and function name from composite key - if ':' in composite_key: - backend, func_name = composite_key.split(':', 1) + if ":" in composite_key: + backend, func_name = composite_key.split(":", 1) else: # Fallback for non-composite keys - backend = metadata.registry.library_name if metadata.registry else 'unknown' + backend = ( + metadata.registry.library_name if metadata.registry else "unknown" + ) func_name = composite_key self.all_functions_metadata[composite_key] = { - 'name': metadata.name, - 'func': metadata.func, - 'module': metadata.module, - 'contract': metadata.contract, - 'tags': metadata.tags, - 'doc': metadata.doc, - 'backend': metadata.get_memory_type(), # Actual memory type (cupy, numpy, etc.) - 'registry': metadata.get_registry_name(), # Registry source (openhcs, skimage, etc.) - 'metadata': metadata # Store full metadata for access to new methods + "name": metadata.name, + "func": metadata.func, + "module": metadata.module, + "contract": metadata.contract, + "tags": metadata.tags, + "doc": metadata.doc, + "backend": metadata.get_memory_type(), # Actual memory type (cupy, numpy, etc.) + "registry": metadata.get_registry_name(), # Registry source (openhcs, skimage, etc.) + "metadata": metadata, # Store full metadata for access to new methods } # Cache the results for future use FunctionSelectorDialog._metadata_cache = self.all_functions_metadata - logger.info(f"Loaded {len(self.all_functions_metadata)} functions from all registries") - - + logger.info( + f"Loaded {len(self.all_functions_metadata)} functions from all registries" + ) self.filtered_functions = self.all_functions_metadata.copy() @@ -214,7 +226,9 @@ def _on_functions_changed(self): self.populate_module_tree() self.populate_function_table() - logger.debug(f"Function selector refreshed with {len(self.all_functions_metadata)} functions") + logger.debug( + f"Function selector refreshed with {len(self.all_functions_metadata)} functions" + ) def populate_module_tree(self): """Populate the module tree with hierarchical function organization based purely on module paths.""" @@ -228,9 +242,13 @@ def populate_module_tree(self): self._add_function_to_hierarchy(module_hierarchy, module_path, func_name) # Build tree structure directly from module hierarchy (no library grouping) - self._build_module_hierarchy_tree(self.module_tree, module_hierarchy, [], is_root=True) + self._build_module_hierarchy_tree( + self.module_tree, module_hierarchy, [], is_root=True + ) - def _update_filtered_view(self, filtered_functions: Dict[str, Any], filter_description: str = ""): + def _update_filtered_view( + self, filtered_functions: Dict[str, Any], filter_description: str = "" + ): """Update filtered view using table browser.""" self.filtered_functions = filtered_functions self.populate_function_table(self.filtered_functions) @@ -275,32 +293,34 @@ def _create_pane_widget(self, title: str, main_widget) -> QWidget: def _determine_library(self, metadata) -> str: """Direct library determination using registry ownership (robust approach).""" - func_name = metadata.get('name', '') - module = metadata.get('module', '') + func_name = metadata.get("name", "") + module = metadata.get("module", "") # Use direct registry ownership lookup (eliminates pattern matching fragility) return LibraryDetector.get_function_library(func_name, module) def _extract_module_path(self, metadata) -> str: """Extract full module path from metadata for hierarchical tree building.""" - module = metadata.get('module', '') + module = metadata.get("module", "") if not module: - return 'unknown' + return "unknown" # Return the full module path for hierarchical tree building return module - def _add_function_to_hierarchy(self, hierarchy: dict, module_path: str, func_name: str): + def _add_function_to_hierarchy( + self, hierarchy: dict, module_path: str, func_name: str + ): """Add a function to the hierarchical module structure.""" - if module_path == 'unknown': + if module_path == "unknown": # Handle unknown modules - if 'functions' not in hierarchy: - hierarchy['functions'] = [] - hierarchy['functions'].append(func_name) + if "functions" not in hierarchy: + hierarchy["functions"] = [] + hierarchy["functions"].append(func_name) return # Split module path and build hierarchy - parts = module_path.split('.') + parts = module_path.split(".") current_level = hierarchy for part in parts: @@ -309,28 +329,35 @@ def _add_function_to_hierarchy(self, hierarchy: dict, module_path: str, func_nam current_level = current_level[part] # Add function to the deepest level - if 'functions' not in current_level: - current_level['functions'] = [] - current_level['functions'].append(func_name) + if "functions" not in current_level: + current_level["functions"] = [] + current_level["functions"].append(func_name) def _count_functions_in_hierarchy(self, hierarchy: dict) -> int: """Count total functions in a hierarchical structure.""" count = 0 for key, value in hierarchy.items(): - if key == 'functions': + if key == "functions": count += len(value) elif isinstance(value, dict): count += self._count_functions_in_hierarchy(value) return count - def _build_module_hierarchy_tree(self, parent_container, hierarchy: dict, - module_path_parts: list, is_root: bool = False): + def _build_module_hierarchy_tree( + self, + parent_container, + hierarchy: dict, + module_path_parts: list, + is_root: bool = False, + ): """Recursively build the hierarchical module tree.""" for key, value in hierarchy.items(): - if key == 'functions': + if key == "functions": # This level has functions - create a module node if there are functions if value: # Only create node if there are functions - current_path = '.'.join(module_path_parts) if module_path_parts else 'unknown' + current_path = ( + ".".join(module_path_parts) if module_path_parts else "unknown" + ) if is_root: # For root level, add directly to tree widget module_item = QTreeWidgetItem(parent_container) @@ -338,11 +365,11 @@ def _build_module_hierarchy_tree(self, parent_container, hierarchy: dict, # For nested levels, add to parent item module_item = QTreeWidgetItem(parent_container) module_item.setText(0, f"{current_path} ({len(value)} functions)") - module_item.setData(0, Qt.ItemDataRole.UserRole, { - "type": "module", - "module": current_path, - "functions": value - }) + module_item.setData( + 0, + Qt.ItemDataRole.UserRole, + {"type": "module", "module": current_path, "functions": value}, + ) elif isinstance(value, dict): # This is a module part - create a tree node and recurse new_path_parts = module_path_parts + [key] @@ -357,17 +384,25 @@ def _build_module_hierarchy_tree(self, parent_container, hierarchy: dict, # For nested levels, add to parent item module_part_item = QTreeWidgetItem(parent_container) - module_part_item.setText(0, f"{key} ({subtree_function_count} functions)") - module_part_item.setData(0, Qt.ItemDataRole.UserRole, { - "type": "module_part", - "module_part": key, - "full_path": '.'.join(new_path_parts) - }) + module_part_item.setText( + 0, f"{key} ({subtree_function_count} functions)" + ) + module_part_item.setData( + 0, + Qt.ItemDataRole.UserRole, + { + "type": "module_part", + "module_part": key, + "full_path": ".".join(new_path_parts), + }, + ) # Start collapsed - users can expand as needed module_part_item.setExpanded(False) # Recursively build subtree - self._build_module_hierarchy_tree(module_part_item, value, new_path_parts, is_root=False) + self._build_module_hierarchy_tree( + module_part_item, value, new_path_parts, is_root=False + ) @classmethod def clear_metadata_cache(cls) -> None: @@ -375,7 +410,7 @@ def clear_metadata_cache(cls) -> None: cls._metadata_cache = None RegistryService.clear_metadata_cache() logger.info("Function metadata cache cleared") - + def setup_ui(self): """Setup the dual-pane user interface with tree, filters, and table.""" self.setWindowTitle("Select Function - Dual Pane View") @@ -391,12 +426,16 @@ def setup_ui(self): title_font.setBold(True) title_font.setPointSize(12) title_label.setFont(title_font) - title_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};") + title_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};" + ) layout.addWidget(title_label) # Create main horizontal splitter (left panel | right table) main_splitter = QSplitter(Qt.Orientation.Horizontal) - main_splitter.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) + main_splitter.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding + ) main_splitter.setHandleWidth(5) # === LEFT PANEL: Tree + Filters === @@ -406,13 +445,19 @@ def setup_ui(self): # Module tree self.module_tree = QTreeWidget() - self.module_tree.setHeaderLabel("Module Structure") - self.module_tree.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) + self.module_tree.setHeaderHidden(True) + self.module_tree.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding + ) self.module_tree.mousePressEvent = self._tree_mouse_press_event - left_layout.addWidget(self._create_pane_widget("Module Structure", self.module_tree), 1) + left_layout.addWidget( + self._create_pane_widget("Module Structure", self.module_tree), 1 + ) # Column filter panel (Library, Backend, Contract, Tags) - self.column_filter_panel = MultiColumnFilterPanel(color_scheme=self.color_scheme) + self.column_filter_panel = MultiColumnFilterPanel( + color_scheme=self.color_scheme + ) self.column_filter_panel.setVisible(False) # Hidden until populated left_layout.addWidget(self.column_filter_panel) @@ -420,12 +465,15 @@ def setup_ui(self): # === RIGHT PANEL: Function Table Browser === self.function_table_browser = FunctionTableBrowser( - color_scheme=self.color_scheme, - parent=self + color_scheme=self.color_scheme, parent=self + ) + self.function_table_browser.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding ) - self.function_table_browser.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) - right_widget = self._create_pane_widget("Function Details", self.function_table_browser) + right_widget = self._create_pane_widget( + "Function Details", self.function_table_browser + ) main_splitter.addWidget(right_widget) # Set splitter proportions @@ -449,16 +497,19 @@ def setup_ui(self): # Apply centralized styling self.setStyleSheet( - self.style_generator.generate_dialog_style() + "\n" + - self.style_generator.generate_tree_widget_style() + "\n" + - self.style_generator.generate_table_widget_style() + "\n" + - self.style_generator.generate_button_style() + self.style_generator.generate_dialog_style() + + "\n" + + self.style_generator.generate_tree_widget_style() + + "\n" + + self.style_generator.generate_table_widget_style() + + "\n" + + self.style_generator.generate_button_style() ) # Connect buttons self.select_btn.clicked.connect(self.accept_selection) cancel_btn.clicked.connect(self.reject) - + def setup_connections(self): """Setup signal/slot connections.""" # Tree selection for filtering @@ -466,12 +517,18 @@ def setup_connections(self): # Table browser signals self.function_table_browser.item_selected.connect(self._on_function_selected) - self.function_table_browser.item_double_clicked.connect(self._on_function_double_clicked) + self.function_table_browser.item_double_clicked.connect( + self._on_function_double_clicked + ) # Column filter panel - self.column_filter_panel.filters_changed.connect(self._on_column_filters_changed) + self.column_filter_panel.filters_changed.connect( + self._on_column_filters_changed + ) - def populate_function_table(self, functions_metadata: Optional[Dict[str, FunctionMetadata]] = None): + def populate_function_table( + self, functions_metadata: Optional[Dict[str, FunctionMetadata]] = None + ): """Populate function table using FunctionTableBrowser.""" if functions_metadata is None: functions_metadata = self.filtered_functions @@ -481,7 +538,9 @@ def populate_function_table(self, functions_metadata: Optional[Dict[str, Functio # Update count label total = len(self.all_functions_metadata) filtered = len(functions_metadata) - self.function_table_browser.status_label.setText(f"Functions: {filtered}/{total}") + self.function_table_browser.status_label.setText( + f"Functions: {filtered}/{total}" + ) def _build_column_filters(self): """Build column filter widgets from function metadata.""" @@ -492,22 +551,25 @@ def _build_column_filters(self): # Extract unique values for filterable columns filter_columns = { - 'Registry': lambda m: m.get('registry', 'unknown').title(), - 'Backend': lambda m: m.get('backend', 'unknown').title(), - 'Contract': lambda m: ( - m.get('contract').name if hasattr(m.get('contract'), 'name') - else str(m.get('contract')) if m.get('contract') else 'unknown' + "Registry": lambda m: m.get("registry", "unknown").title(), + "Backend": lambda m: m.get("backend", "unknown").title(), + "Contract": lambda m: ( + m.get("contract").name + if hasattr(m.get("contract"), "name") + else str(m.get("contract")) + if m.get("contract") + else "unknown" ), - 'Tags': None, # Special handling for tags (multiple values per item) + "Tags": None, # Special handling for tags (multiple values per item) } for column_name, extractor in filter_columns.items(): unique_values = set() for metadata in self.all_functions_metadata.values(): - if column_name == 'Tags': + if column_name == "Tags": # Tags is a list - add each tag individually - tags = metadata.get('tags', []) + tags = metadata.get("tags", []) unique_values.update(tags) else: value = extractor(metadata) @@ -515,7 +577,9 @@ def _build_column_filters(self): unique_values.add(value) if unique_values: - self.column_filter_panel.add_column_filter(column_name, sorted(list(unique_values))) + self.column_filter_panel.add_column_filter( + column_name, sorted(list(unique_values)) + ) if self.column_filter_panel.column_filters: self.column_filter_panel.setVisible(True) @@ -535,16 +599,22 @@ def _on_column_filters_changed(self): matches = True for column_name, allowed_values in active_filters.items(): - if column_name == 'Registry': - value = metadata.get('registry', 'unknown').title() - elif column_name == 'Backend': - value = metadata.get('backend', 'unknown').title() - elif column_name == 'Contract': - contract = metadata.get('contract') - value = contract.name if hasattr(contract, 'name') else str(contract) if contract else 'unknown' - elif column_name == 'Tags': + if column_name == "Registry": + value = metadata.get("registry", "unknown").title() + elif column_name == "Backend": + value = metadata.get("backend", "unknown").title() + elif column_name == "Contract": + contract = metadata.get("contract") + value = ( + contract.name + if hasattr(contract, "name") + else str(contract) + if contract + else "unknown" + ) + elif column_name == "Tags": # For tags, match if ANY tag is in allowed_values - tags = metadata.get('tags', []) + tags = metadata.get("tags", []) if not any(tag in allowed_values for tag in tags): matches = False continue @@ -568,7 +638,9 @@ def on_tree_selection_changed(self): # If no items selected, show all functions if not selected_items: - self._update_filtered_view(self.all_functions_metadata, "showing all functions") + self._update_filtered_view( + self.all_functions_metadata, "showing all functions" + ) return item = selected_items[0] @@ -581,7 +653,8 @@ def on_tree_selection_changed(self): if node_type == "module": module_functions = data.get("functions", []) filtered = { - name: metadata for name, metadata in self.all_functions_metadata.items() + name: metadata + for name, metadata in self.all_functions_metadata.items() if name in module_functions } self._update_filtered_view(filtered, "filtered by module") @@ -590,13 +663,18 @@ def on_tree_selection_changed(self): # Filter by module part - find all functions whose modules start with this path module_part_path = data.get("full_path", "") filtered = { - name: metadata for name, metadata in self.all_functions_metadata.items() + name: metadata + for name, metadata in self.all_functions_metadata.items() if self._extract_module_path(metadata).startswith(module_part_path) } - self._update_filtered_view(filtered, f"filtered by module part: {module_part_path}") + self._update_filtered_view( + filtered, f"filtered by module part: {module_part_path}" + ) else: # No data means show all functions - self._update_filtered_view(self.all_functions_metadata, "showing all functions") + self._update_filtered_view( + self.all_functions_metadata, "showing all functions" + ) def _tree_mouse_press_event(self, event): """Handle mouse press events on the tree to allow deselection.""" @@ -612,35 +690,37 @@ def _tree_mouse_press_event(self, event): def _on_function_selected(self, key: str, item: Dict[str, Any]): """Handle function selection from table browser.""" - func = item.get('func') + func = item.get("func") self._set_selection_state(func, func is not None) def _on_function_double_clicked(self, key: str, item: Dict[str, Any]): """Handle function double-click from table browser.""" - func = item.get('func') + func = item.get("func") if func: self.selected_function = func self.accept_selection() - + def accept_selection(self): """Accept the selected function.""" if self.selected_function: self.function_selected.emit(self.selected_function) self.accept() - + def get_selected_function(self) -> Optional[Callable]: """Get the selected function.""" return self.selected_function - + @staticmethod - def select_function(current_function: Optional[Callable] = None, parent=None) -> Optional[Callable]: + def select_function( + current_function: Optional[Callable] = None, parent=None + ) -> Optional[Callable]: """ Static method to show function selector and return selected function. - + Args: current_function: Currently selected function (for highlighting) parent: Parent widget - + Returns: Selected function or None if cancelled """ diff --git a/openhcs/pyqt_gui/dialogs/metadata_viewer_dialog.py b/openhcs/pyqt_gui/dialogs/metadata_viewer_dialog.py index fc8701b5f..e105c503f 100644 --- a/openhcs/pyqt_gui/dialogs/metadata_viewer_dialog.py +++ b/openhcs/pyqt_gui/dialogs/metadata_viewer_dialog.py @@ -9,15 +9,21 @@ from typing import Optional from PyQt6.QtWidgets import ( - QVBoxLayout, QHBoxLayout, QPushButton, - QScrollArea, QWidget, QLabel, QGroupBox + QVBoxLayout, + QHBoxLayout, + QPushButton, + QScrollArea, + QWidget, + QLabel, + QGroupBox, + QComboBox, ) from PyQt6.QtCore import Qt from openhcs.microscopes.openhcs import OpenHCSMetadata from pyqt_reactive.forms import ParameterFormManager from pyqt_reactive.theming import ColorScheme -from openhcs.pyqt_gui.windows.base_form_dialog import BaseFormDialog +from pyqt_reactive.widgets.shared import BaseFormDialog logger = logging.getLogger(__name__) @@ -33,7 +39,9 @@ class MetadataViewerDialog(BaseFormDialog): Only ONE MetadataViewerDialog per plate can be open at a time. """ - def __init__(self, orchestrator, color_scheme: Optional[ColorScheme] = None, parent=None): + def __init__( + self, orchestrator, color_scheme: Optional[ColorScheme] = None, parent=None + ): """ Initialize metadata viewer dialog. @@ -48,50 +56,50 @@ def __init__(self, orchestrator, color_scheme: Optional[ColorScheme] = None, par # scope_id for singleton behavior - one viewer per plate self.scope_id = str(orchestrator.plate_path) if orchestrator else None - + self.setWindowTitle(f"Plate Metadata - {orchestrator.plate_path.name}") self.setMinimumSize(800, 600) self.resize(1000, 700) - + # Make floating like other OpenHCS windows self.setWindowFlags(Qt.WindowType.Dialog) self._setup_ui() self._load_metadata() - + def _setup_ui(self): """Setup the dialog UI.""" layout = QVBoxLayout(self) layout.setContentsMargins(5, 5, 5, 5) # Reduced margins layout.setSpacing(5) # Reduced spacing - + # Title label title_label = QLabel(f"Plate: {self.orchestrator.plate_path}") title_label.setWordWrap(True) layout.addWidget(title_label) - + # Scroll area for metadata form scroll_area = QScrollArea() scroll_area.setWidgetResizable(True) scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) - + # Container for the form self.form_container = QWidget() scroll_area.setWidget(self.form_container) layout.addWidget(scroll_area) - + # Button row button_layout = QHBoxLayout() button_layout.addStretch() - + close_button = QPushButton("Close") close_button.clicked.connect(self.accept) close_button.setMinimumWidth(100) button_layout.addWidget(close_button) - + layout.addLayout(button_layout) - + def _load_metadata(self): """Load and display metadata using generic reflection.""" try: @@ -99,7 +107,7 @@ def _load_metadata(self): plate_path = self.orchestrator.plate_path # Check if this is OpenHCS format (has subdirectory-keyed metadata) - if hasattr(metadata_handler, '_load_metadata_dict'): + if hasattr(metadata_handler, "_load_metadata_dict"): # OpenHCS format - read subdirectory-keyed metadata metadata_dict = metadata_handler._load_metadata_dict(plate_path) subdirs_dict = metadata_dict.get("subdirectories", {}) @@ -107,42 +115,67 @@ def _load_metadata(self): if not subdirs_dict: raise ValueError("No subdirectories found in metadata") + def ensure_optional_fields(subdir_data): + subdir_data.setdefault("timepoints", None) + subdir_data.setdefault("channels", None) + subdir_data.setdefault("wells", None) + subdir_data.setdefault("sites", None) + subdir_data.setdefault("z_indexes", None) + # Create metadata instance based on number of subdirectories if len(subdirs_dict) == 1: # Single subdirectory - show OpenHCSMetadata directly subdir_name = next(iter(subdirs_dict.keys())) subdir_data = subdirs_dict[subdir_name] - # Ensure all optional fields have explicit None if missing - subdir_data.setdefault('timepoints', None) - subdir_data.setdefault('channels', None) - subdir_data.setdefault('wells', None) - subdir_data.setdefault('sites', None) - subdir_data.setdefault('z_indexes', None) + ensure_optional_fields(subdir_data) metadata_instance = OpenHCSMetadata(**subdir_data) window_title = f"Metadata - {subdir_name}" self._create_single_metadata_form(metadata_instance) else: - # Multiple subdirectories - manually create forms for each - # ParameterFormManager can't handle Dict[str, dataclass], so we create forms manually - subdirs_instances = {} - for name, data in subdirs_dict.items(): - # Ensure all optional fields have explicit None if missing - data.setdefault('timepoints', None) - data.setdefault('channels', None) - data.setdefault('wells', None) - data.setdefault('sites', None) - data.setdefault('z_indexes', None) - subdirs_instances[name] = OpenHCSMetadata(**data) + # Multiple subdirectories - select one at a time window_title = f"Metadata - {len(subdirs_dict)} subdirectories" - self._create_multi_subdirectory_forms(subdirs_instances) + + selector_row = QHBoxLayout() + selector_label = QLabel("Subdirectory:") + selector = QComboBox() + selector.addItems(sorted(subdirs_dict.keys())) + selector_row.addWidget(selector_label) + selector_row.addWidget(selector, 1) + + form_layout = QVBoxLayout(self.form_container) + form_layout.setContentsMargins(5, 5, 5, 5) + form_layout.addLayout(selector_row) + + def clear_layout(target_layout): + while target_layout.count() > 1: + item = target_layout.takeAt(1) + widget = item.widget() + if widget is not None: + widget.deleteLater() + + def render_selected(subdir_name): + clear_layout(form_layout) + subdir_data = subdirs_dict[subdir_name] + ensure_optional_fields(subdir_data) + metadata_instance = OpenHCSMetadata(**subdir_data) + self._create_single_metadata_form( + metadata_instance, layout=form_layout + ) + + selector.currentTextChanged.connect(render_selected) + render_selected(selector.currentText()) else: # Other microscope formats - build metadata from handler methods # Use parse_metadata() to get component mappings component_metadata = metadata_handler.parse_metadata(plate_path) # Get basic metadata using handler methods - grid_dims = metadata_handler._get_with_fallback('get_grid_dimensions', plate_path) - pixel_size = metadata_handler._get_with_fallback('get_pixel_size', plate_path) + grid_dims = metadata_handler._get_with_fallback( + "get_grid_dimensions", plate_path + ) + pixel_size = metadata_handler._get_with_fallback( + "get_pixel_size", plate_path + ) # Build OpenHCSMetadata structure # Note: For non-OpenHCS formats, we don't have image_files list or available_backends @@ -152,25 +185,27 @@ def _load_metadata(self): grid_dimensions=list(grid_dims) if grid_dims else [1, 1], pixel_size=pixel_size if pixel_size else 1.0, image_files=[], # Not available for non-OpenHCS formats - channels=component_metadata.get('channel'), - wells=component_metadata.get('well'), - sites=component_metadata.get('site'), - z_indexes=component_metadata.get('z_index'), - timepoints=component_metadata.get('timepoint'), - available_backends={'disk': True}, # Assume disk backend - main=None + channels=component_metadata.get("channel"), + wells=component_metadata.get("well"), + sites=component_metadata.get("site"), + z_indexes=component_metadata.get("z_index"), + timepoints=component_metadata.get("timepoint"), + available_backends={"disk": True}, # Assume disk backend + main=None, + ) + window_title = ( + f"Metadata - {self.orchestrator.microscope_handler.microscope_type}" ) - window_title = f"Metadata - {self.orchestrator.microscope_handler.microscope_type}" self._create_single_metadata_form(metadata_instance) # Update window title self.setWindowTitle(f"{window_title} - {self.orchestrator.plate_path.name}") - + logger.info(f"Loaded metadata for {self.orchestrator.plate_path}") - + except Exception as e: logger.error(f"Failed to load metadata: {e}", exc_info=True) - + # Show error in form container error_layout = QVBoxLayout(self.form_container) error_label = QLabel(f"Error loading metadata:
{str(e)}") @@ -179,13 +214,20 @@ def _load_metadata(self): error_layout.addWidget(error_label) error_layout.addStretch() - def _create_single_metadata_form(self, metadata_instance: OpenHCSMetadata): + def _create_single_metadata_form( + self, metadata_instance: OpenHCSMetadata, layout: Optional[QVBoxLayout] = None + ): """Create a single metadata form for one OpenHCSMetadata instance.""" from pyqt_reactive.forms import FormManagerConfig from openhcs.config_framework.object_state import ObjectState - form_layout = QVBoxLayout(self.form_container) - form_layout.setContentsMargins(5, 5, 5, 5) + form_layout = layout or QVBoxLayout(self.form_container) + if layout is None: + form_layout.setContentsMargins(5, 5, 5, 5) + + image_files = getattr(metadata_instance, "image_files", None) + if image_files is not None: + form_layout.addWidget(QLabel(f"Image files: {len(image_files)} (hidden)")) # Create local ObjectState for metadata viewer state = ObjectState( @@ -198,8 +240,9 @@ def _create_single_metadata_form(self, metadata_instance: OpenHCSMetadata): state=state, config=FormManagerConfig( parent=self.form_container, - read_only=True - ) + read_only=True, + exclude_params=["image_files", "workspace_mapping"], + ), ) form_layout.addWidget(metadata_form) @@ -240,6 +283,12 @@ def _create_multi_subdirectory_forms(self, subdirs_instances: dict): group_layout = QVBoxLayout(group_box) group_layout.setContentsMargins(10, 10, 10, 10) + image_files = getattr(metadata_instance, "image_files", None) + if image_files is not None: + group_layout.addWidget( + QLabel(f"Image files: {len(image_files)} (hidden)") + ) + # Create local ObjectState for this subdirectory's metadata state = ObjectState( object_instance=metadata_instance, @@ -251,12 +300,12 @@ def _create_multi_subdirectory_forms(self, subdirs_instances: dict): state=state, config=FormManagerConfig( parent=group_box, - read_only=True - ) + read_only=True, + exclude_params=["image_files", "workspace_mapping"], + ), ) group_layout.addWidget(metadata_form) form_layout.addWidget(group_box) form_layout.addStretch() - diff --git a/openhcs/pyqt_gui/launch.py b/openhcs/pyqt_gui/launch.py index 65d3f622d..196207af3 100644 --- a/openhcs/pyqt_gui/launch.py +++ b/openhcs/pyqt_gui/launch.py @@ -24,18 +24,6 @@ root_logger = logging.getLogger() root_logger.setLevel(logging.CRITICAL + 1) -# Add OpenHCS to path if needed -try: - from openhcs.core.config import GlobalPipelineConfig -except ImportError: - # Add parent directory to path - sys.path.insert(0, str(Path(__file__).parent.parent.parent)) - from openhcs.core.config import GlobalPipelineConfig - - -from openhcs.pyqt_gui.app import OpenHCSPyQtApp -from pyqt_reactive.utils.window_utils import install_global_window_bounds_filter - def is_wsl() -> bool: """Check if running in Windows Subsystem for Linux.""" @@ -46,7 +34,21 @@ def is_wsl() -> bool: def setup_qt_platform(): - """Setup Qt platform for different environments (macOS, Linux, WSL2, Windows).""" + """Configure Qt platform based on OS and display environment. + + Qt requires different platform plugins for different operating systems and + display servers. This function detects the current environment and sets + QT_QPA_PLATFORM accordingly, with special handling for WSL2's dual-mode + support (Wayland via WSLg on Windows 11+, or X11/xcb fallback). + + Detection order for WSL2: + 1. Wayland (WSLg on Windows 11+) + 2. X11 via DISPLAY variable + 3. Fallback to auto-detect (wayland;xcb) + + The environment variables must be set BEFORE any PyQt6 imports occur, + as Qt caches platform selection at import time. + """ import platform from pathlib import Path @@ -76,12 +78,29 @@ def setup_qt_platform(): logging.warning(f"Could not set QT_QPA_PLATFORM_PLUGIN_PATH: {e}") elif platform.system() == 'Linux': - os.environ['QT_QPA_PLATFORM'] = 'xcb' if is_wsl(): - logging.info("WSL2 detected - setting QT_QPA_PLATFORM=xcb") + logging.info("WSL2 detected") + # WSL2 can use Wayland (WSLg on Windows 11+) or X11. Detect which is available + # and configure Qt accordingly. Wayland is preferred as it provides better + # integration with Windows 11's native graphics. + display = os.environ.get('DISPLAY') + wayland_display = os.environ.get('WAYLAND_DISPLAY') + + if wayland_display: + os.environ['QT_QPA_PLATFORM'] = 'wayland' + logging.info("WSL2+WSLg with Wayland detected - setting QT_QPA_PLATFORM=wayland") + elif display: + os.environ['QT_QPA_PLATFORM'] = 'xcb' + logging.info(f"WSL2 with X11 display {display} - setting QT_QPA_PLATFORM=xcb") + else: + # Neither display server explicitly configured. Let Qt auto-detect in order. + os.environ['QT_QPA_PLATFORM'] = 'wayland;xcb' + logging.info("WSL2 detected - enabling platform auto-detection (wayland;xcb)") else: + os.environ['QT_QPA_PLATFORM'] = 'xcb' logging.info("Linux detected - setting QT_QPA_PLATFORM=xcb") - # Disable shared memory for X11 (helps with display issues) + + # Disable X11 shared memory (MITSHM) for compatibility. os.environ['QT_X11_NO_MITSHM'] = '1' # Windows doesn't need QT_QPA_PLATFORM set else: @@ -295,9 +314,14 @@ def main(): logging.info(f"Python version: {sys.version}") logging.info(f"Platform: {sys.platform}") - # Setup Qt platform (must be done before creating QApplication) + # Setup Qt platform (must be done BEFORE any OpenHCS imports that depend on PyQt6) setup_qt_platform() + # NOW import OpenHCS modules and dependencies that may load PyQt6 + from openhcs.core.config import GlobalPipelineConfig + from openhcs.pyqt_gui.app import OpenHCSPyQtApp + from pyqt_reactor.utils.window_utils import install_global_window_bounds_filter + try: # Check dependencies if not check_dependencies(): diff --git a/openhcs/pyqt_gui/main.py b/openhcs/pyqt_gui/main.py index 13c590dd3..1310c6756 100644 --- a/openhcs/pyqt_gui/main.py +++ b/openhcs/pyqt_gui/main.py @@ -1,32 +1,48 @@ """ OpenHCS PyQt6 Main Window -Main application window implementing QDockWidget system to replace -textual-window floating windows with native Qt docking. +Main application window using WindowManager for clean window abstraction. """ import logging -from typing import Optional, Dict +from typing import Optional, Callable from pathlib import Path import webbrowser from PyQt6.QtWidgets import ( - QMainWindow, QWidget, QVBoxLayout, - QMessageBox, QFileDialog, QDialog + QApplication, + QMainWindow, + QWidget, + QVBoxLayout, + QMessageBox, + QFileDialog, + QDialog, + QProgressBar, ) from PyQt6.QtCore import Qt, QSettings, QTimer, pyqtSignal, QUrl -from PyQt6.QtGui import QAction, QKeySequence, QDesktopServices +from PyQt6.QtGui import QAction, QKeySequence, QDesktopServices, QShowEvent from openhcs.core.config import GlobalPipelineConfig from polystore.filemanager import FileManager from polystore.base import storage_registry from openhcs.pyqt_gui.services.service_adapter import PyQtServiceAdapter +from openhcs.pyqt_gui.services.window_config import WindowSpec +from openhcs.config_framework.object_state import ObjectState from pyqt_reactive.animation.flash_overlay_opengl import prewarm_opengl from pyqt_reactive.animation import WindowFlashOverlay +from pyqt_reactive.services.window_manager import WindowManager from pyqt_reactive.widgets.log_viewer import LogViewerWindow from pyqt_reactive.widgets.system_monitor import SystemMonitorWidget from pyqt_reactive.widgets.editors.simple_code_editor import QScintillaCodeEditorDialog +from openhcs.pyqt_gui.services.time_travel_navigation import ( + TimeTravelNavigationTarget, + parse_function_scope_ref, + make_function_token_target, + make_field_path_target, + resolve_fallback_field_path, + should_replace_navigation_target, +) logger = logging.getLogger(__name__) @@ -34,15 +50,15 @@ class OpenHCSMainWindow(QMainWindow): """ Main OpenHCS PyQt6 application window. - + Implements QDockWidget system to replace textual-window floating windows with native Qt docking, providing better desktop integration. """ - + # Signals for application events config_changed = pyqtSignal(object) # GlobalPipelineConfig status_message = pyqtSignal(str) # Status message - + def __init__(self, global_config: Optional[GlobalPipelineConfig] = None): """ Initialize the main OpenHCS window. @@ -54,16 +70,19 @@ def __init__(self, global_config: Optional[GlobalPipelineConfig] = None): # Core configuration self.global_config = global_config or GlobalPipelineConfig() - + # Create shared components self.storage_registry = storage_registry self.file_manager = FileManager(self.storage_registry) - + # Service adapter for Qt integration self.service_adapter = PyQtServiceAdapter(self) - - # Floating windows registry (replaces dock widgets) - self.floating_windows: Dict[str, QDialog] = {} + + # Declarative window specs + self.window_specs = self._get_window_specs() + + # Track any floating windows created outside WindowManager + self.create_floating_windows() # Settings for window state persistence self.settings = QSettings("OpenHCS", "PyQt6GUI") @@ -73,8 +92,6 @@ def __init__(self, global_config: Optional[GlobalPipelineConfig] = None): # Initialize UI self.setup_ui() - self.setup_dock_system() - self.create_floating_windows() self.setup_menu_bar() self.setup_status_bar() self.setup_connections() @@ -85,7 +102,9 @@ def __init__(self, global_config: Optional[GlobalPipelineConfig] = None): # Restore window state self.restore_window_state() - logger.info("OpenHCS PyQt6 main window initialized (deferred initialization pending)") + logger.info( + "OpenHCS PyQt6 main window initialized (deferred initialization pending)" + ) def _deferred_initialization(self): """ @@ -97,35 +116,186 @@ def _deferred_initialization(self): Note: System monitor is now created during __init__ so startup screen appears immediately """ - # Initialize Log Viewer (hidden) for continuous log monitoring - IMMEDIATE - self._initialize_log_viewer() + # Initialize log viewer (hidden) for continuous log monitoring - IMMEDIATE + self.show_window("log_viewer") # Show default windows (plate manager and pipeline editor visible by default) - IMMEDIATE self.show_default_windows() logger.info("Deferred initialization complete (UI ready)") + def _get_window_specs(self) -> dict[str, WindowSpec]: + """Return declarative window specifications.""" + from openhcs.pyqt_gui.windows.managed_windows import ( + PlateManagerWindow, + PipelineEditorWindow, + ImageBrowserWindow, + LogViewerWindowWrapper, + ZMQServerManagerWindow, + ) + return { + "plate_manager": WindowSpec( + window_id="plate_manager", + title="Plate Manager", + window_class=PlateManagerWindow, + initialize_on_startup=True, + ), + "pipeline_editor": WindowSpec( + window_id="pipeline_editor", + title="Pipeline Editor", + window_class=PipelineEditorWindow, + ), + "image_browser": WindowSpec( + window_id="image_browser", + title="Image Browser", + window_class=ImageBrowserWindow, + ), + "log_viewer": WindowSpec( + window_id="log_viewer", + title="Log Viewer", + window_class=LogViewerWindowWrapper, + initialize_on_startup=True, + ), + "zmq_server_manager": WindowSpec( + window_id="zmq_server_manager", + title="ZMQ Server Manager", + window_class=ZMQServerManagerWindow, + ), + } + + def _create_window_factory(self, window_id: str) -> Callable[[], QDialog]: + """Create factory function for a window.""" + spec = self.window_specs[window_id] + + def factory() -> QDialog: + window = spec.window_class(self, self.service_adapter) + return window + + return factory + + def show_window(self, window_id: str, hide_if_startup: bool = True) -> None: + """Show window using WindowManager.""" + factory = self._create_window_factory(window_id) + window = WindowManager.show_or_focus(window_id, factory) + + spec = self.window_specs[window_id] + if hide_if_startup and spec.initialize_on_startup and window_id == "log_viewer": + window.hide() + + self._ensure_flash_overlay(window) def setup_ui(self): """Setup basic UI structure.""" self.setWindowTitle("OpenHCS") - self.setMinimumSize(640, 480) + self.setMinimumSize(1024, 768) + self.resize(self.minimumSize()) # Make main window floating (not tiled) like other OpenHCS components self.setWindowFlags(Qt.WindowType.Dialog) - # Central widget with system monitor (shows startup screen immediately) + # Central widget with main layout central_widget = QWidget() central_layout = QVBoxLayout(central_widget) central_layout.setContentsMargins(0, 0, 0, 0) + central_layout.setSpacing(0) + + # Main vertical splitter: System Monitor (top) vs rest (bottom) + from PyQt6.QtWidgets import QSplitter + + top_splitter = QSplitter(Qt.Orientation.Vertical) - # Create system monitor immediately so startup screen shows right away + # Top section: System Monitor self.system_monitor = SystemMonitorWidget() - central_layout.addWidget(self.system_monitor) + top_splitter.addWidget(self.system_monitor) + + # Connect system monitor button signals to main window actions + self.system_monitor.show_global_config.connect(self.show_configuration) + self.system_monitor.show_log_viewer.connect(self.show_log_viewer) + self.system_monitor.show_custom_functions.connect( + self._on_manage_custom_functions + ) + self.system_monitor.show_test_plate_generator.connect( + self.show_synthetic_plate_generator + ) + + # Bottom section: Main horizontal splitter + main_splitter = QSplitter(Qt.Orientation.Horizontal) + + # LEFT SIDE: Vertical splitter with Plate Manager (top) and ZMQ Browser (bottom) + left_splitter = QSplitter(Qt.Orientation.Vertical) + + # Plate Manager (top of left side) + # Auto-registers with ServiceRegistry via AutoRegisterServiceMixin + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + + self.plate_manager_widget = PlateManagerWidget( + self.service_adapter, self.service_adapter.get_current_color_scheme() + ) + left_splitter.addWidget(self.plate_manager_widget) + + # ZMQ Server Manager (bottom of left side) + from openhcs.pyqt_gui.widgets.shared.zmq_server_manager import ( + ZMQServerManagerWidget, + ) + from openhcs.core.config import get_all_streaming_ports + + ports_to_scan = get_all_streaming_ports(num_ports_per_type=10) + self.zmq_manager_widget = ZMQServerManagerWidget( + ports_to_scan=ports_to_scan, + title="ZMQ Servers", + style_generator=self.service_adapter.get_style_generator(), + ) + self.zmq_manager_widget.log_file_opened.connect(self._open_log_file_in_viewer) + left_splitter.addWidget(self.zmq_manager_widget) + + # Set sizes for left splitter (70% plate manager, 30% ZMQ) + left_splitter.setSizes([350, 150]) + + main_splitter.addWidget(left_splitter) + + # RIGHT SIDE: Pipeline Editor + from openhcs.pyqt_gui.widgets.pipeline_editor import PipelineEditorWidget + + self.pipeline_editor_widget = PipelineEditorWidget( + self.service_adapter, self.service_adapter.get_current_color_scheme() + ) + main_splitter.addWidget(self.pipeline_editor_widget) + + # Connect plate manager to pipeline editor (mirrors Textual TUI and ManagedWindow pattern) + self.plate_manager_widget.plate_selected.connect( + self.pipeline_editor_widget.set_current_plate + ) + self.plate_manager_widget.orchestrator_config_changed.connect( + self.pipeline_editor_widget.on_orchestrator_config_changed + ) + self.plate_manager_widget.set_pipeline_editor(self.pipeline_editor_widget) + self.pipeline_editor_widget.plate_manager = self.plate_manager_widget + + # Set current plate if one is already selected + if self.plate_manager_widget.selected_plate_path: + self.pipeline_editor_widget.set_current_plate( + self.plate_manager_widget.selected_plate_path + ) - # Store layout for potential future use + # Set sizes for main splitter (50/50) + main_splitter.setSizes([500, 500]) + + # Add main splitter to top splitter (bottom section) + top_splitter.addWidget(main_splitter) + + # Set sizes for top splitter - system monitor takes minimum, bottom takes all available space + top_splitter.setSizes([1, 1000]) + top_splitter.setStretchFactor(0, 0) # System monitor doesn't stretch + top_splitter.setStretchFactor(1, 1) # Bottom section takes all available space + + central_layout.addWidget(top_splitter, 1) # Stretch to fill remaining space + + # Store references self.central_layout = central_layout + self.top_splitter = top_splitter + self.main_splitter = main_splitter + self.left_splitter = left_splitter self.setCentralWidget(central_widget) @@ -138,7 +308,9 @@ def apply_initial_theme(self): # Just register for theme change notifications, don't re-apply theme_manager.register_theme_change_callback(self.on_theme_changed) - logger.debug("Registered for theme change notifications (theme already applied by ServiceAdapter)") + logger.debug( + "Registered for theme change notifications (theme already applied by ServiceAdapter)" + ) def on_theme_changed(self, color_scheme): """ @@ -150,14 +322,14 @@ def on_theme_changed(self, color_scheme): # Update any main window specific styling if needed # Most styling is handled automatically by the theme manager logger.debug("Main window received theme change notification") - + def setup_dock_system(self): """Setup window system mirroring Textual TUI floating windows.""" # In Textual TUI, widgets are floating windows, not docked # We'll create windows on-demand when menu items are clicked # Only the system monitor stays as the central background widget pass - + def create_floating_windows(self): """Create floating windows mirroring Textual TUI window system.""" # Windows are created on-demand when menu items are clicked @@ -170,218 +342,76 @@ def _ensure_flash_overlay(self, window: QWidget) -> None: def show_default_windows(self): """Show plate manager by default.""" - # Show plate manager by default - self.show_plate_manager() + # Plate Manager and Pipeline Editor are now embedded in the main window + # Just ensure they're visible (in case they were hidden) + if hasattr(self, "plate_manager_widget"): + self.plate_manager_widget.show() + if hasattr(self, "pipeline_editor_widget"): + self.pipeline_editor_widget.show() - # Pipeline editor is NOT shown by default because it imports ALL GPU libraries - # (torch, tensorflow, jax, cupy, pyclesperanto) which takes 8+ seconds - # User can open it from View menu when needed + # Log viewer is still a separate window (on-demand) def show_plate_manager(self): - """Show plate manager window (mirrors Textual TUI pattern).""" - if "plate_manager" not in self.floating_windows: - from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget - - # Create floating window - window = QDialog(self) - window.setWindowTitle("Plate Manager") - window.setModal(False) - window.resize(600, 400) - - # Add widget to window - layout = QVBoxLayout(window) - plate_widget = PlateManagerWidget( - self.service_adapter, - self.service_adapter.get_current_color_scheme() - ) - layout.addWidget(plate_widget) - - self.floating_windows["plate_manager"] = window - - # REACTIVITY: Connect global config changed signal so windows auto-refresh - # When PlateManager saves global config, it emits this signal - # Main window propagates it to all other windows via on_config_changed - plate_widget.global_config_changed.connect(lambda: self.on_config_changed(self.service_adapter.get_global_config())) - logger.debug("Connected PlateManager global_config_changed signal for reactive updates") - - # Connect progress signals to status bar - if hasattr(self, 'status_bar') and self.status_bar: - # Create progress bar in status bar if it doesn't exist - if not hasattr(self, '_status_progress_bar'): - from PyQt6.QtWidgets import QProgressBar - self._status_progress_bar = QProgressBar() - self._status_progress_bar.setMaximumWidth(200) - self._status_progress_bar.setVisible(False) - self.status_bar.addPermanentWidget(self._status_progress_bar) - - # Connect progress signals - plate_widget.progress_started.connect( - lambda max_val: self._on_plate_progress_started(max_val) - ) - plate_widget.progress_updated.connect( - lambda val: self._on_plate_progress_updated(val) - ) - plate_widget.progress_finished.connect( - lambda: self._on_plate_progress_finished() - ) - - # Connect to pipeline editor if it exists (mirrors Textual TUI) - self._connect_plate_to_pipeline_manager(plate_widget) - - # Show the window and ensure flash overlay is ready - self.floating_windows["plate_manager"].show() - self._ensure_flash_overlay(self.floating_windows["plate_manager"]) - self.floating_windows["plate_manager"].raise_() - self.floating_windows["plate_manager"].activateWindow() + """Show plate manager widget if not already visible.""" + if hasattr(self, "plate_manager_widget"): + if not self.plate_manager_widget.isVisible(): + self.plate_manager_widget.show() + # Only resize on first show + if hasattr(self, "left_splitter"): + sizes = self.left_splitter.sizes() + if len(sizes) == 2: + total = sum(sizes) + self.left_splitter.setSizes([int(total * 0.7), int(total * 0.3)]) + if hasattr(self, "main_splitter"): + sizes = self.main_splitter.sizes() + if len(sizes) == 2: + total = sum(sizes) + self.main_splitter.setSizes([int(total * 0.6), int(total * 0.4)]) + else: + self.show_window("plate_manager") def show_pipeline_editor(self): - """Show pipeline editor window (mirrors Textual TUI pattern).""" - if "pipeline_editor" not in self.floating_windows: - from openhcs.pyqt_gui.widgets.pipeline_editor import PipelineEditorWidget - - # Create floating window - window = QDialog(self) - window.setWindowTitle("Pipeline Editor") - window.setModal(False) - window.resize(800, 600) - - # Add widget to window - layout = QVBoxLayout(window) - pipeline_widget = PipelineEditorWidget( - self.service_adapter, - self.service_adapter.get_current_color_scheme() - ) - layout.addWidget(pipeline_widget) - - self.floating_windows["pipeline_editor"] = window - - # Connect to plate manager for current plate selection (mirrors Textual TUI) - self._connect_pipeline_to_plate_manager(pipeline_widget) + """Show pipeline editor widget if not already visible.""" + if hasattr(self, "pipeline_editor_widget"): + if not self.pipeline_editor_widget.isVisible(): + self.pipeline_editor_widget.show() + if hasattr(self, "main_splitter"): + sizes = self.main_splitter.sizes() + if len(sizes) == 2: + total = sum(sizes) + self.main_splitter.setSizes([int(total * 0.4), int(total * 0.6)]) + else: + self.show_window("pipeline_editor") - # Show the window and ensure flash overlay is ready - self.floating_windows["pipeline_editor"].show() - self._ensure_flash_overlay(self.floating_windows["pipeline_editor"]) - self.floating_windows["pipeline_editor"].raise_() - self.floating_windows["pipeline_editor"].activateWindow() + def show_zmq_server_manager(self): + """Show ZMQ server manager widget if not already visible.""" + if hasattr(self, "zmq_manager_widget"): + if not self.zmq_manager_widget.isVisible(): + self.zmq_manager_widget.show() + if hasattr(self, "left_splitter"): + sizes = self.left_splitter.sizes() + if len(sizes) == 2: + total = sum(sizes) + self.left_splitter.setSizes([int(total * 0.5), int(total * 0.5)]) + if hasattr(self, "main_splitter"): + sizes = self.main_splitter.sizes() + if len(sizes) == 2: + total = sum(sizes) + self.main_splitter.setSizes([int(total * 0.6), int(total * 0.4)]) + else: + self.show_window("zmq_server_manager") def show_image_browser(self): """Show image browser window.""" - if "image_browser" not in self.floating_windows: - from openhcs.pyqt_gui.widgets.image_browser import ImageBrowserWidget - from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget - - # Create floating window - window = QDialog(self) - window.setWindowTitle("Image Browser") - window.setModal(False) - window.resize(900, 600) - - # Add widget to window - layout = QVBoxLayout(window) - image_browser_widget = ImageBrowserWidget( - orchestrator=None, - color_scheme=self.service_adapter.get_current_color_scheme() - ) - layout.addWidget(image_browser_widget) - - self.floating_windows["image_browser"] = window - - # Connect to plate manager to get current orchestrator - if "plate_manager" in self.floating_windows: - plate_dialog = self.floating_windows["plate_manager"] - plate_widget = plate_dialog.findChild(PlateManagerWidget) - if plate_widget: - # Connect to plate selection changes - def on_plate_selected(): - orchestrator = plate_widget.get_selected_orchestrator() - if orchestrator: - image_browser_widget.set_orchestrator(orchestrator) - - # Connect to plate selection signal - plate_widget.plate_selected.connect(on_plate_selected) - - # Set initial orchestrator if available - on_plate_selected() - - # Show the window and ensure flash overlay is ready - self.floating_windows["image_browser"].show() - self._ensure_flash_overlay(self.floating_windows["image_browser"]) - self.floating_windows["image_browser"].raise_() - self.floating_windows["image_browser"].activateWindow() - - def _initialize_log_viewer(self): - """ - Initialize Log Viewer on startup (hidden) for continuous log monitoring. - - This ensures all server logs are captured regardless of when the - Log Viewer window is opened by the user. - """ - # Create floating window (hidden) - window = QDialog(self) - window.setWindowTitle("Log Viewer") - window.setModal(False) - window.resize(900, 700) - - # Add widget to window - layout = QVBoxLayout(window) - log_viewer_widget = LogViewerWindow(self.file_manager, self.service_adapter) - layout.addWidget(log_viewer_widget) - - self.floating_windows["log_viewer"] = window - - # Window stays hidden until user opens it - logger.info("Log Viewer initialized (hidden) - monitoring for new logs") + self.show_window("image_browser") def show_log_viewer(self): - """Show log viewer window (mirrors Textual TUI pattern).""" - # Log viewer is already initialized on startup, just show it - if "log_viewer" in self.floating_windows: - self.floating_windows["log_viewer"].show() - self._ensure_flash_overlay(self.floating_windows["log_viewer"]) - self.floating_windows["log_viewer"].raise_() - self.floating_windows["log_viewer"].activateWindow() - else: - # Fallback: initialize if somehow not created - self._initialize_log_viewer() - self.show_log_viewer() + """Show log viewer window.""" + self.show_window("log_viewer", hide_if_startup=False) def show_zmq_server_manager(self): """Show ZMQ server manager window.""" - if "zmq_server_manager" not in self.floating_windows: - from openhcs.pyqt_gui.widgets.shared.zmq_server_manager import ZMQServerManagerWidget - - # Create floating window - window = QDialog(self) - window.setWindowTitle("ZMQ Server Manager") - window.setModal(False) - window.resize(600, 400) - - # Add widget to window - layout = QVBoxLayout(window) - - # Scan all streaming ports using current global config - # This ensures we find viewers launched with custom ports - from openhcs.core.config import get_all_streaming_ports - ports_to_scan = get_all_streaming_ports(num_ports_per_type=10) # Uses global config by default - - zmq_manager_widget = ZMQServerManagerWidget( - ports_to_scan=ports_to_scan, - title="ZMQ Servers (Execution + Napari + Fiji)", - style_generator=self.service_adapter.get_style_generator() - ) - - # Connect log file opened signal to log viewer - zmq_manager_widget.log_file_opened.connect(self._open_log_file_in_viewer) - - layout.addWidget(zmq_manager_widget) - - self.floating_windows["zmq_server_manager"] = window - - # Show window and ensure flash overlay is ready - self.floating_windows["zmq_server_manager"].show() - self._ensure_flash_overlay(self.floating_windows["zmq_server_manager"]) - self.floating_windows["zmq_server_manager"].raise_() - self.floating_windows["zmq_server_manager"].activateWindow() + self.show_window("zmq_server_manager") def _open_log_file_in_viewer(self, log_file_path: str): """ @@ -390,10 +420,13 @@ def _open_log_file_in_viewer(self, log_file_path: str): Args: log_file_path: Path to log file to open """ - # Show log viewer if not already open self.show_log_viewer() - # Switch to the log file + window = self._get_managed_window("log_viewer") + if window: + log_viewer_widget = window.get_widget() + log_viewer_widget.switch_to_log(Path(log_file_path)) + logger.info(f"Switched log viewer to: {log_file_path}") if "log_viewer" in self.floating_windows: log_dialog = self.floating_windows["log_viewer"] log_viewer_widget = log_dialog.findChild(LogViewerWindow) @@ -408,8 +441,8 @@ def setup_menu_bar(self): # File menu file_menu = menubar.addMenu("&File") - - # Theme submenu + + # Theme submenu theme_menu = file_menu.addMenu("&Theme") # Dark theme action @@ -433,9 +466,9 @@ def setup_menu_bar(self): save_theme_action = QAction("&Save Theme to File...", self) save_theme_action.triggered.connect(self.save_theme_to_file) theme_menu.addAction(save_theme_action) - + file_menu.addSeparator() - + # Exit action exit_action = QAction("E&xit", self) exit_action.setShortcut(QKeySequence.StandardKey.Quit) @@ -444,6 +477,7 @@ def setup_menu_bar(self): # View menu - shortcuts come from declarative ShortcutConfig from openhcs.pyqt_gui.config import get_shortcut_config + shortcuts = get_shortcut_config() view_menu = menubar.addMenu("&View") @@ -524,7 +558,9 @@ def setup_menu_bar(self): analysis_menu.addAction(merge_summaries_action) # Concatenate MetaXpress Summaries (keep all headers) action - concat_summaries_action = QAction("&Concatenate MetaXpress Summaries (Keep Headers)...", self) + concat_summaries_action = QAction( + "&Concatenate MetaXpress Summaries (Keep Headers)...", self + ) concat_summaries_action.triggered.connect(self._on_concat_metaxpress_summaries) analysis_menu.addAction(concat_summaries_action) @@ -532,7 +568,9 @@ def setup_menu_bar(self): # Run Experimental Analysis action experimental_analysis_action = QAction("Run &Experimental Analysis...", self) - experimental_analysis_action.triggered.connect(self._on_run_experimental_analysis) + experimental_analysis_action.triggered.connect( + self._on_run_experimental_analysis + ) analysis_menu.addAction(experimental_analysis_action) # Help menu @@ -544,7 +582,6 @@ def setup_menu_bar(self): help_action.triggered.connect(self.show_help) help_menu.addAction(help_action) - def setup_status_bar(self): """Setup application status bar.""" self.status_bar = self.statusBar() @@ -552,19 +589,14 @@ def setup_status_bar(self): # Add time-travel widget to LEFT side of status bar # Note: Don't use showMessage() as it hides addWidget() widgets from openhcs.pyqt_gui.widgets.shared.time_travel_widget import TimeTravelWidget + color_scheme = self.service_adapter.get_current_color_scheme() self.time_travel_widget = TimeTravelWidget(color_scheme=color_scheme) self.status_bar.addWidget(self.time_travel_widget) - # Add graph layout toggle button to the right side of status bar - # Only add if system monitor widget exists and has the method - if hasattr(self, 'system_monitor') and hasattr(self.system_monitor, 'create_layout_toggle_button'): - toggle_button = self.system_monitor.create_layout_toggle_button() - self.status_bar.addPermanentWidget(toggle_button) - # Connect status message signal self.status_message.connect(self.status_bar.showMessage) - + def setup_connections(self): """Setup signal/slot connections.""" # Connect config changes @@ -580,20 +612,40 @@ def setup_connections(self): # Subscribe to time-travel completion to reopen windows for dirty states from openhcs.config_framework.object_state import ObjectStateRegistry - ObjectStateRegistry.add_time_travel_complete_callback(self._on_time_travel_complete) + + ObjectStateRegistry.add_time_travel_complete_callback( + self._on_time_travel_complete + ) # Subscribe to ObjectState unregistration to auto-close associated windows # This ensures windows close when time-traveling removes their backing state ObjectStateRegistry.add_unregister_callback(self._on_object_state_unregistered) - # Register OpenHCS window factory for scope-aware window creation - from pyqt_reactive.protocols import register_window_factory - from openhcs.pyqt_gui.services.window_factory import OpenHCSWindowFactory - register_window_factory(OpenHCSWindowFactory()) + # Register OpenHCS window handlers with the generic factory + from openhcs.pyqt_gui.services.window_handlers import ( + register_openhcs_window_handlers, + ) + + register_openhcs_window_handlers() # Setup global keyboard shortcuts from declarative config self._setup_global_shortcuts() + def showEvent(self, event: QShowEvent) -> None: + super().showEvent(event) + self._log_window_size("shown") + + def _log_window_size(self, context: str) -> None: + size = self.size() + logger.info( + "Main window %s size=%dx%d pos=%d,%d", + context, + size.width(), + size.height(), + self.x(), + self.y(), + ) + def _setup_global_shortcuts(self): """Setup global keyboard shortcuts from declarative ShortcutConfig. @@ -611,17 +663,17 @@ def _setup_global_shortcuts(self): # Time travel functions def time_travel_back(): ObjectStateRegistry.time_travel_back() - if hasattr(self, 'time_travel_widget'): + if hasattr(self, "time_travel_widget"): self.time_travel_widget.refresh() def time_travel_forward(): ObjectStateRegistry.time_travel_forward() - if hasattr(self, 'time_travel_widget'): + if hasattr(self, "time_travel_widget"): self.time_travel_widget.refresh() def time_travel_to_head(): ObjectStateRegistry.time_travel_to_head() - if hasattr(self, 'time_travel_widget'): + if hasattr(self, "time_travel_widget"): self.time_travel_widget.refresh() # Store actions for event filter @@ -664,6 +716,21 @@ def eventFilter(filter_self, obj, event): if key_str: full_key = mod_str + key_str if full_key in filter_self.main_window._time_travel_actions: + # Check if focused widget is a code editor - let it handle Ctrl+Z/Y + from PyQt6.QtWidgets import QApplication + from PyQt6.Qsci import QsciScintilla + + focused = QApplication.focusWidget() + if focused is not None: + # Check if focused widget or its parent is a QsciScintilla editor + widget = focused + while widget is not None: + if isinstance(widget, QsciScintilla): + # Let code editor handle Ctrl+Z/Y for text undo/redo + return False + widget = widget.parentWidget() + + # Not in a code editor - handle time travel filter_self.main_window._time_travel_actions[full_key]() return True # Consume event - don't pass to widget @@ -672,59 +739,60 @@ def eventFilter(filter_self, obj, event): self._event_filter = TimeTravelEventFilter(self) QApplication.instance().installEventFilter(self._event_filter) - logger.info(f"Global shortcuts (event filter): {shortcuts.time_travel_back.key}=back, " - f"{shortcuts.time_travel_forward.key}=forward, " - f"{shortcuts.time_travel_to_head.key}=head") + logger.info( + f"Global shortcuts (event filter): {shortcuts.time_travel_back.key}=back, " + f"{shortcuts.time_travel_forward.key}=forward, " + f"{shortcuts.time_travel_to_head.key}=head" + ) def restore_window_state(self): """Restore window state from settings.""" + logger.debug("Skipping window state restore; persistence disabled") + return try: geometry = self.settings.value("geometry") if geometry: self.restoreGeometry(geometry) - + window_state = self.settings.value("windowState") if window_state: self.restoreState(window_state) - + except Exception as e: logger.warning(f"Failed to restore window state: {e}") - + def save_window_state(self): """Save window state to settings.""" # Skip settings save for now to prevent hanging # TODO: Investigate QSettings hanging issue logger.debug("Skipping window state save to prevent hanging") - + # Menu action handlers def new_pipeline(self): """Create new pipeline.""" if "pipeline_editor" in self.dock_widgets: pipeline_widget = self.dock_widgets["pipeline_editor"].widget() - if hasattr(pipeline_widget, 'new_pipeline'): + if hasattr(pipeline_widget, "new_pipeline"): pipeline_widget.new_pipeline() - + def open_pipeline(self): """Open existing pipeline.""" file_path, _ = QFileDialog.getOpenFileName( - self, - "Open Pipeline", - "", - "Function Files (*.func);;All Files (*)" + self, "Open Pipeline", "", "Function Files (*.func);;All Files (*)" ) - + if file_path and "pipeline_editor" in self.dock_widgets: pipeline_widget = self.dock_widgets["pipeline_editor"].widget() - if hasattr(pipeline_widget, 'load_pipeline'): + if hasattr(pipeline_widget, "load_pipeline"): pipeline_widget.load_pipeline(Path(file_path)) - + def save_pipeline(self): """Save current pipeline.""" if "pipeline_editor" in self.dock_widgets: pipeline_widget = self.dock_widgets["pipeline_editor"].widget() - if hasattr(pipeline_widget, 'save_pipeline'): + if hasattr(pipeline_widget, "save_pipeline"): pipeline_widget.save_pipeline() - + def show_configuration(self): """Show configuration dialog for global config editing.""" from openhcs.pyqt_gui.windows.config_window import ConfigWindow @@ -736,7 +804,10 @@ def handle_config_save(new_config): # Update thread-local storage for MaterializationPathConfig defaults from openhcs.core.config import GlobalPipelineConfig - from openhcs.config_framework.global_config import set_global_config_for_editing + from openhcs.config_framework.global_config import ( + set_global_config_for_editing, + ) + set_global_config_for_editing(GlobalPipelineConfig, new_config) # Emit signal for other components to update @@ -751,10 +822,10 @@ def handle_config_save(new_config): config_window = ConfigWindow( GlobalPipelineConfig, # config_class (concrete class for static context) self.service_adapter.get_global_config(), # current_config (concrete instance) - handle_config_save, # on_save_callback + handle_config_save, # on_save_callback self.service_adapter.get_current_color_scheme(), # color_scheme - self, # parent - scope_id="" # Global scope - matches app.py registration + self, # parent + scope_id="", # Global scope - matches app.py registration ) # Show as non-modal window (like plate manager and pipeline editor) config_window.show() @@ -764,74 +835,78 @@ def handle_config_save(new_config): def _connect_pipeline_to_plate_manager(self, pipeline_widget): """Connect pipeline editor to plate manager (mirrors Textual TUI pattern).""" from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + from pyqt_reactive.services import ServiceRegistry - # Get plate manager if it exists - if "plate_manager" in self.floating_windows: - plate_manager_window = self.floating_windows["plate_manager"] - - # Find the actual plate manager widget using type-based dispatch - plate_manager_widget = plate_manager_window.findChild(PlateManagerWidget) + # Get plate manager from ServiceRegistry + plate_manager_widget = ServiceRegistry.get(PlateManagerWidget) - if plate_manager_widget: - # Connect plate selection signal to pipeline editor (mirrors Textual TUI) - plate_manager_widget.plate_selected.connect(pipeline_widget.set_current_plate) + if plate_manager_widget: + # Connect plate selection signal to pipeline editor (mirrors Textual TUI) + plate_manager_widget.plate_selected.connect( + pipeline_widget.set_current_plate + ) - # Connect orchestrator config changed signal for placeholder refresh - plate_manager_widget.orchestrator_config_changed.connect(pipeline_widget.on_orchestrator_config_changed) + # Connect orchestrator config changed signal for placeholder refresh + plate_manager_widget.orchestrator_config_changed.connect( + pipeline_widget.on_orchestrator_config_changed + ) - # Set pipeline editor reference in plate manager - plate_manager_widget.set_pipeline_editor(pipeline_widget) + # Set pipeline editor reference in plate manager + plate_manager_widget.set_pipeline_editor(pipeline_widget) - # Set plate manager reference in pipeline editor (for step editor signal connections) - pipeline_widget.plate_manager = plate_manager_widget + # Set plate manager reference in pipeline editor (for step editor signal connections) + pipeline_widget.plate_manager = plate_manager_widget - # Set current plate if one is already selected - if plate_manager_widget.selected_plate_path: - pipeline_widget.set_current_plate(plate_manager_widget.selected_plate_path) + # Set current plate if one is already selected + if plate_manager_widget.selected_plate_path: + pipeline_widget.set_current_plate( + plate_manager_widget.selected_plate_path + ) - logger.debug("Connected pipeline editor to plate manager") - else: - logger.warning("Could not find plate manager widget to connect") + logger.debug("Connected pipeline editor to plate manager") else: - logger.debug("Plate manager not yet created - connection will be made when both exist") + logger.warning("Could not find plate manager widget to connect") def _connect_plate_to_pipeline_manager(self, plate_manager_widget): """Connect plate manager to pipeline editor (reverse direction).""" from openhcs.pyqt_gui.widgets.pipeline_editor import PipelineEditorWidget + from pyqt_reactive.services import ServiceRegistry - # Get pipeline editor if it exists - if "pipeline_editor" in self.floating_windows: - pipeline_editor_window = self.floating_windows["pipeline_editor"] - - # Find the actual pipeline editor widget using type-based dispatch - pipeline_editor_widget = pipeline_editor_window.findChild(PipelineEditorWidget) + # Get pipeline editor from ServiceRegistry + pipeline_editor_widget = ServiceRegistry.get(PipelineEditorWidget) - if pipeline_editor_widget: - # Connect plate selection signal to pipeline editor (mirrors Textual TUI) - logger.info(f"🔗 CONNECTING plate_selected signal to pipeline editor") - plate_manager_widget.plate_selected.connect(pipeline_editor_widget.set_current_plate) + if pipeline_editor_widget: + # Connect plate selection signal to pipeline editor (mirrors Textual TUI) + logger.info(f"🔗 CONNECTING plate_selected signal to pipeline editor") + plate_manager_widget.plate_selected.connect( + pipeline_editor_widget.set_current_plate + ) - # Connect orchestrator config changed signal for placeholder refresh - plate_manager_widget.orchestrator_config_changed.connect(pipeline_editor_widget.on_orchestrator_config_changed) + # Connect orchestrator config changed signal for placeholder refresh + plate_manager_widget.orchestrator_config_changed.connect( + pipeline_editor_widget.on_orchestrator_config_changed + ) - # Set pipeline editor reference in plate manager - plate_manager_widget.set_pipeline_editor(pipeline_editor_widget) + # Set pipeline editor reference in plate manager + plate_manager_widget.set_pipeline_editor(pipeline_editor_widget) - # Set plate manager reference in pipeline editor (for step editor signal connections) - pipeline_editor_widget.plate_manager = plate_manager_widget + # Set plate manager reference in pipeline editor (for step editor signal connections) + pipeline_editor_widget.plate_manager = plate_manager_widget - # Set current plate if one is already selected - if plate_manager_widget.selected_plate_path: - logger.info(f"🔗 Setting initial plate: {plate_manager_widget.selected_plate_path}") - pipeline_editor_widget.set_current_plate(plate_manager_widget.selected_plate_path) + # Set current plate if one is already selected + if plate_manager_widget.selected_plate_path: + logger.info( + f"🔗 Setting initial plate: {plate_manager_widget.selected_plate_path}" + ) + pipeline_editor_widget.set_current_plate( + plate_manager_widget.selected_plate_path + ) - logger.info("✅ Connected plate manager to pipeline editor") - else: - logger.warning("Could not find pipeline editor widget to connect") + logger.info("✅ Connected plate manager to pipeline editor") else: - logger.debug("Pipeline editor not yet created - connection will be made when both exist") + logger.warning("Could not find pipeline editor widget to connect") - def _on_object_state_unregistered(self, scope_id: str, state: 'ObjectState'): + def _on_object_state_unregistered(self, scope_id: str, state: "ObjectState"): """Handle ObjectState unregistration by closing associated windows. When time-travel removes a step/config (unregisters its ObjectState), @@ -846,7 +921,9 @@ def _on_object_state_unregistered(self, scope_id: str, state: 'ObjectState'): if WindowManager.is_open(scope_id): WindowManager.close_window(scope_id) - logger.info(f"⏱️ TIME_TRAVEL: Auto-closed window for unregistered state: {scope_id}") + logger.info( + f"⏱️ TIME_TRAVEL: Auto-closed window for unregistered state: {scope_id}" + ) def _on_time_travel_complete(self, dirty_states, triggering_scope: str | None): """Handle time-travel completion by reopening windows for dirty ObjectStates. @@ -863,28 +940,115 @@ def _on_time_travel_complete(self, dirty_states, triggering_scope: str | None): """ from pyqt_reactive.services.window_manager import WindowManager - logger.debug(f"⏱️ TIME_TRAVEL_CALLBACK: triggering_scope={triggering_scope!r} dirty_count={len(dirty_states)}") + logger.debug( + f"⏱️ TIME_TRAVEL_CALLBACK: triggering_scope={triggering_scope!r} dirty_count={len(dirty_states)}" + ) - for scope_id, state in dirty_states: - # Check if window is already open - if WindowManager.is_open(scope_id): - # Window exists - just focus it - WindowManager.focus_and_navigate(scope_id) + from objectstate import ObjectStateRegistry + + # Consolidate navigation requests: + # - Function ObjectStates map to their parent step scope + # - Function-scope changes navigate with token payload so function editor can + # choose the correct dict-pattern key on its own invariant path. + pending: dict[str, TimeTravelNavigationTarget | None] = {} + + for entry in dirty_states: + if not isinstance(entry, (tuple, list)) or len(entry) != 2: + continue + scope_id, state = entry + if not isinstance(scope_id, str) or not isinstance(state, ObjectState): continue + use_scope_id = scope_id + use_state = state + target: TimeTravelNavigationTarget | None = None + + # If this is a function ObjectState, map to parent step scope and + # force Function Pattern tab with token payload. + function_scope = parse_function_scope_ref(scope_id) + if function_scope is not None: + use_scope_id = function_scope.step_scope_id + parent_state = ObjectStateRegistry.get_by_scope(use_scope_id) + if isinstance(parent_state, ObjectState): + use_state = parent_state + target = make_function_token_target(function_scope.function_token) + + if target is None: + # Use last_changed_field - tracks ANY value change (clean->dirty OR dirty->clean) + # This shows what changed in the time-travel transition, not just current dirty state + field_path = resolve_fallback_field_path( + use_state.last_changed_field, use_state.dirty_fields + ) + logger.debug( + f"⏱️ TIME_TRAVEL_NAV: scope={use_scope_id} last_changed_field={field_path}" + ) + if field_path: + target = make_field_path_target(field_path) + + existing = pending.get(use_scope_id) + if existing is None: + pending[use_scope_id] = target + else: + if should_replace_navigation_target(existing, target): + pending[use_scope_id] = target + + for scope_id, target in pending.items(): + field_path = target.to_field_path() if target is not None else None + if WindowManager.is_open(scope_id): + self._select_tab_for_time_travel(scope_id, target) + # Window exists - focus and navigate to dirty field + WindowManager.focus_and_navigate(scope_id, field_path=field_path) + else: + # Create new window + from pyqt_reactive.services import WindowFactory + + window = WindowFactory.create_window_for_scope(scope_id, state) + if window: + logger.info( + f"⏱️ TIME_TRAVEL: Reopened window for dirty state: {scope_id}" + ) + self._select_tab_for_time_travel(scope_id, target) + if field_path: + WindowManager.focus_and_navigate( + scope_id, field_path=field_path + ) + + def _select_tab_for_time_travel( + self, scope_id: str, target: TimeTravelNavigationTarget | None + ) -> None: + """Select appropriate tab in step editor after time-travel. + + If 'func' parameter was modified, switch to Function Pattern tab. + Otherwise, stay on Step Settings tab. + """ + from pyqt_reactive.services.window_manager import WindowManager + from openhcs.pyqt_gui.windows.dual_editor_window import DualEditorWindow - # Window not open - create via shared infrastructure - window = WindowManager.create_window_for_scope(scope_id, state) - if window: - logger.info(f"⏱️ TIME_TRAVEL: Reopened window for dirty state: {scope_id}") + window = WindowManager.get_window(scope_id) + if not isinstance(window, DualEditorWindow): + return + + if window.tab_widget is None: + return + + is_function_scope = parse_function_scope_ref(scope_id) is not None + if is_function_scope or ( + target is not None and target.is_function_target + ): + window.tab_widget.setCurrentIndex(1) + logger.debug("[TAB_SELECT] Time-travel: Function Pattern tab") + else: + window.tab_widget.setCurrentIndex(0) + logger.debug("[TAB_SELECT] Time-travel: Step Settings tab") def show_synthetic_plate_generator(self): """Show synthetic plate generator window.""" - from openhcs.pyqt_gui.windows.synthetic_plate_generator_window import SyntheticPlateGeneratorWindow + from openhcs.pyqt_gui.windows.synthetic_plate_generator_window import ( + SyntheticPlateGeneratorWindow, + ) # Create and show the generator window generator_window = SyntheticPlateGeneratorWindow( - color_scheme=self.service_adapter.get_current_color_scheme(), - parent=self + color_scheme=self.service_adapter.get_current_color_scheme(), parent=self ) # Connect the plate_generated signal to add the plate to the manager @@ -911,13 +1075,14 @@ def _on_synthetic_plate_generated(self, output_dir: str, pipeline_path: str): # This ensures the pipeline is saved to plate_pipelines[plate_path] self._load_pipeline_file(pipeline_path, plate_path=output_dir) - # Get the plate manager widget - plate_dialog = self.floating_windows["plate_manager"] + # Get the plate manager widget from ServiceRegistry from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget - plate_manager = plate_dialog.findChild(PlateManagerWidget) + from pyqt_reactive.services import ServiceRegistry + + plate_manager = ServiceRegistry.get(PlateManagerWidget) if not plate_manager: - raise RuntimeError("Plate manager widget not found after creation") + raise RuntimeError("Plate manager widget not found in ServiceRegistry") # Add the generated plate - this triggers plate_selected signal # which automatically updates pipeline editor via existing connections @@ -938,30 +1103,36 @@ def _load_pipeline_file(self, pipeline_path: str, plate_path: str = None): # Ensure pipeline editor exists (create if needed) self.show_pipeline_editor() - # Get the pipeline editor widget - pipeline_dialog = self.floating_windows["pipeline_editor"] + # Get the pipeline editor widget from ServiceRegistry from openhcs.pyqt_gui.widgets.pipeline_editor import PipelineEditorWidget - pipeline_editor = pipeline_dialog.findChild(PipelineEditorWidget) + from pyqt_reactive.services import ServiceRegistry + + pipeline_editor = ServiceRegistry.get(PipelineEditorWidget) if not pipeline_editor: - raise RuntimeError("Pipeline editor widget not found after creation") + raise RuntimeError( + "Pipeline editor widget not found in ServiceRegistry" + ) # If plate_path is provided, set it as current_plate BEFORE loading # This ensures _apply_executed_code() can save to plate_pipelines[current_plate] if plate_path: pipeline_editor.current_plate = plate_path - logger.debug(f"Set current_plate to {plate_path} before loading pipeline") + logger.debug( + f"Set current_plate to {plate_path} before loading pipeline" + ) # Load the pipeline file from pathlib import Path + pipeline_file = Path(pipeline_path) if not pipeline_file.exists(): raise FileNotFoundError(f"Pipeline file not found: {pipeline_path}") # For .py files, read code and use existing _handle_edited_code - if pipeline_file.suffix == '.py': - with open(pipeline_file, 'r') as f: + if pipeline_file.suffix == ".py": + with open(pipeline_file, "r") as f: code = f.read() # Use existing infrastructure that already handles code execution @@ -976,8 +1147,6 @@ def _load_pipeline_file(self, pipeline_path: str, plate_path: str = None): logger.error(f"Failed to load pipeline: {e}", exc_info=True) raise - - def _on_consolidate_results(self): """Open file dialog to select results directory and consolidate analysis results.""" from PyQt6.QtWidgets import QFileDialog, QMessageBox @@ -985,10 +1154,7 @@ def _on_consolidate_results(self): # Select results directory results_dir = QFileDialog.getExistingDirectory( - self, - "Select Results Directory", - "", - QFileDialog.Option.ShowDirsOnly + self, "Select Results Directory", "", QFileDialog.Option.ShowDirsOnly ) if not results_dir: @@ -998,21 +1164,25 @@ def _on_consolidate_results(self): # Check for CSV files csv_files = list(results_path.glob("*.csv")) - csv_files = [f for f in csv_files if not any( - pattern in f.name.lower() - for pattern in ['metaxpress', 'summary', 'consolidated', 'global'] - )] + csv_files = [ + f + for f in csv_files + if not any( + pattern in f.name.lower() + for pattern in ["metaxpress", "summary", "consolidated", "global"] + ) + ] if not csv_files: QMessageBox.warning( - self, - "No CSV Files", - f"No CSV files found in:\n{results_dir}" + self, "No CSV Files", f"No CSV files found in:\n{results_dir}" ) return try: - from openhcs.processing.backends.analysis.consolidate_analysis_results import consolidate_analysis_results + from openhcs.processing.backends.analysis.consolidate_analysis_results import ( + consolidate_analysis_results, + ) from openhcs.config_framework.global_config import get_current_global_config from openhcs.core.config import GlobalPipelineConfig @@ -1023,7 +1193,8 @@ def _on_consolidate_results(self): well_ids = set() for csv_file in csv_files: import re - match = re.search(r'([A-Z]\d{2})', csv_file.name, re.IGNORECASE) + + match = re.search(r"([A-Z]\d{2})", csv_file.name, re.IGNORECASE) if match: well_ids.add(match.group(1).upper()) @@ -1033,7 +1204,7 @@ def _on_consolidate_results(self): QMessageBox.warning( self, "No Wells Found", - f"Could not extract well IDs from CSV filenames in:\n{results_dir}" + f"Could not extract well IDs from CSV filenames in:\n{results_dir}", ) return @@ -1042,16 +1213,19 @@ def _on_consolidate_results(self): results_directory=str(results_path), well_ids=well_ids, consolidation_config=global_config.analysis_consolidation_config, - plate_metadata_config=global_config.plate_metadata_config + plate_metadata_config=global_config.plate_metadata_config, ) - output_file = results_path / global_config.analysis_consolidation_config.output_filename + output_file = ( + results_path + / global_config.analysis_consolidation_config.output_filename + ) QMessageBox.information( self, "Consolidation Complete", f"Successfully consolidated {len(csv_files)} CSV files from {len(well_ids)} wells.\n\n" - f"Output: {output_file.name}" + f"Output: {output_file.name}", ) except Exception as e: @@ -1059,7 +1233,7 @@ def _on_consolidate_results(self): QMessageBox.critical( self, "Consolidation Failed", - f"Failed to consolidate results:\n\n{str(e)}" + f"Failed to consolidate results:\n\n{str(e)}", ) def _on_merge_metaxpress_summaries(self): @@ -1071,7 +1245,9 @@ def _on_merge_metaxpress_summaries(self): file_dialog = QFileDialog(self) file_dialog.setFileMode(QFileDialog.FileMode.ExistingFiles) file_dialog.setNameFilter("MetaXpress CSV (*.csv)") - file_dialog.setWindowTitle("Select MetaXpress Summary Files to Merge (Concat Rows)") + file_dialog.setWindowTitle( + "Select MetaXpress Summary Files to Merge (Concat Rows)" + ) if not file_dialog.exec(): return @@ -1085,14 +1261,16 @@ def _on_merge_metaxpress_summaries(self): self, "Save Merged Summary As", "merged_metaxpress_summary.csv", - "CSV Files (*.csv)" + "CSV Files (*.csv)", ) if not output_file: return try: - from openhcs.processing.backends.analysis.consolidate_analysis_results import consolidate_multi_plate_summaries + from openhcs.processing.backends.analysis.consolidate_analysis_results import ( + consolidate_multi_plate_summaries, + ) # Extract plate names from file paths (parent directory name) plate_names = [Path(f).parent.name for f in selected_files] @@ -1101,21 +1279,19 @@ def _on_merge_metaxpress_summaries(self): consolidate_multi_plate_summaries( summary_paths=selected_files, output_path=output_file, - plate_names=plate_names + plate_names=plate_names, ) QMessageBox.information( self, "Merge Complete", - f"Successfully merged {len(selected_files)} summaries into:\n{output_file}" + f"Successfully merged {len(selected_files)} summaries into:\n{output_file}", ) except Exception as e: logger.error(f"Failed to merge summaries: {e}", exc_info=True) QMessageBox.critical( - self, - "Merge Failed", - f"Failed to merge summaries:\n\n{str(e)}" + self, "Merge Failed", f"Failed to merge summaries:\n\n{str(e)}" ) def _on_concat_metaxpress_summaries(self): @@ -1127,7 +1303,9 @@ def _on_concat_metaxpress_summaries(self): file_dialog = QFileDialog(self) file_dialog.setFileMode(QFileDialog.FileMode.ExistingFiles) file_dialog.setNameFilter("MetaXpress CSV (*.csv)") - file_dialog.setWindowTitle("Select MetaXpress Summary Files to Concatenate (Keep All Headers)") + file_dialog.setWindowTitle( + "Select MetaXpress Summary Files to Concatenate (Keep All Headers)" + ) if not file_dialog.exec(): return @@ -1141,7 +1319,7 @@ def _on_concat_metaxpress_summaries(self): self, "Save Concatenated Summary As", "concatenated_metaxpress_summary.csv", - "CSV Files (*.csv)" + "CSV Files (*.csv)", ) if not output_file: @@ -1149,19 +1327,19 @@ def _on_concat_metaxpress_summaries(self): try: # Read all files and concatenate with headers - with open(output_file, 'w') as outfile: + with open(output_file, "w") as outfile: for i, input_file in enumerate(selected_files): - with open(input_file, 'r') as infile: + with open(input_file, "r") as infile: content = infile.read() outfile.write(content) # Add blank line between files (except after last file) if i < len(selected_files) - 1: - outfile.write('\n') + outfile.write("\n") QMessageBox.information( self, "Concatenation Complete", - f"Successfully concatenated {len(selected_files)} summaries (with all headers) into:\n{output_file}" + f"Successfully concatenated {len(selected_files)} summaries (with all headers) into:\n{output_file}", ) except Exception as e: @@ -1169,7 +1347,7 @@ def _on_concat_metaxpress_summaries(self): QMessageBox.critical( self, "Concatenation Failed", - f"Failed to concatenate summaries:\n\n{str(e)}" + f"Failed to concatenate summaries:\n\n{str(e)}", ) def _on_run_experimental_analysis(self): @@ -1182,7 +1360,7 @@ def _on_run_experimental_analysis(self): self, "Select Experimental Analysis Directory", "", - QFileDialog.Option.ShowDirsOnly + QFileDialog.Option.ShowDirsOnly, ) if not analysis_dir: @@ -1197,7 +1375,7 @@ def _on_run_experimental_analysis(self): QMessageBox.warning( self, "Config File Missing", - f"Expected config.xlsx not found in:\n{analysis_dir}" + f"Expected config.xlsx not found in:\n{analysis_dir}", ) return @@ -1205,7 +1383,7 @@ def _on_run_experimental_analysis(self): QMessageBox.warning( self, "Results File Missing", - f"Expected metaxpress_style_summary.csv not found in:\n{analysis_dir}" + f"Expected metaxpress_style_summary.csv not found in:\n{analysis_dir}", ) return @@ -1221,7 +1399,7 @@ def _on_run_experimental_analysis(self): results_path=str(results_file), config_file=str(config_file), compiled_results_path=str(compiled_results), - heatmap_path=str(heatmaps) + heatmap_path=str(heatmaps), ) QMessageBox.information( @@ -1229,7 +1407,7 @@ def _on_run_experimental_analysis(self): "Analysis Complete", f"Experimental analysis complete!\n\n" f"Compiled results: {compiled_results.name}\n" - f"Heatmaps: {heatmaps.name}" + f"Heatmaps: {heatmaps.name}", ) except Exception as e: @@ -1237,37 +1415,45 @@ def _on_run_experimental_analysis(self): QMessageBox.critical( self, "Analysis Failed", - f"Failed to run experimental analysis:\n\n{str(e)}" + f"Failed to run experimental analysis:\n\n{str(e)}", ) def show_help(self): """Opens documentation URL in default web browser.""" from openhcs.constants.constants import DOCUMENTATION_URL - url = (DOCUMENTATION_URL) + url = DOCUMENTATION_URL if not QDesktopServices.openUrl(QUrl.fromUserInput(url)): - #fallback for wsl users because it wants to be special + # fallback for wsl users because it wants to be special webbrowser.open(url) - - + def on_config_changed(self, new_config: GlobalPipelineConfig): """Handle global configuration changes.""" self.global_config = new_config self.service_adapter.set_global_config(new_config) + # Propagate to embedded core widgets that cache global config locally. + # Without this, batch compile/execute can keep using stale values + # (for example num_workers) even after global config edits. + self.plate_manager_widget.on_config_changed(new_config) + self.pipeline_editor_widget.on_config_changed(new_config) + # Notify all floating windows of config change for window in self.floating_windows.values(): # Get the widget from the window's layout layout = window.layout() widget = layout.itemAt(0).widget() # Only call on_config_changed if the widget has this method - if hasattr(widget, 'on_config_changed'): + if hasattr(widget, "on_config_changed"): widget.on_config_changed(new_config) def _save_config_to_cache(self, config): """Save config to cache asynchronously (matches TUI pattern).""" try: - from openhcs.pyqt_gui.services.config_cache_adapter import get_global_config_cache + from openhcs.pyqt_gui.services.config_cache_adapter import ( + get_global_config_cache, + ) + cache = get_global_config_cache() cache.save_config_to_cache_async(config) logger.info("Global config save to cache initiated") @@ -1280,17 +1466,28 @@ def closeEvent(self, event): try: # Stop system monitor first with timeout - if hasattr(self, 'system_monitor'): + if hasattr(self, "system_monitor"): logger.info("Stopping system monitor...") self.system_monitor.stop_monitoring() + # Close WindowManager-managed windows + from pyqt_reactive.services.window_manager import WindowManager + + for scope_id in WindowManager.get_open_scopes(): + try: + WindowManager.close_window(scope_id) + except Exception as e: + logger.warning(f"Error closing managed window {scope_id}: {e}") + # Close floating windows and cleanup their resources - for window_name, window in list(self.floating_windows.items()): + for window_name, window in list( + getattr(self, "floating_windows", {}).items() + ): try: layout = window.layout() if layout and layout.count() > 0: widget = layout.itemAt(0).widget() - if hasattr(widget, 'cleanup'): + if hasattr(widget, "cleanup"): widget.cleanup() window.close() window.deleteLater() @@ -1298,17 +1495,27 @@ def closeEvent(self, event): logger.warning(f"Error cleaning up window {window_name}: {e}") # Clear floating windows dict - self.floating_windows.clear() + if hasattr(self, "floating_windows"): + self.floating_windows.clear() + + # Close any remaining top-level windows + for widget in QApplication.topLevelWidgets(): + if widget is self: + continue + try: + widget.close() + except Exception as e: + logger.warning(f"Error closing top-level widget: {e}") # Save window state self.save_window_state() # Force Qt to process pending events before shutdown - from PyQt6.QtWidgets import QApplication QApplication.processEvents() # Additional cleanup - force garbage collection import gc + gc.collect() except Exception as e: @@ -1320,6 +1527,7 @@ def closeEvent(self, event): # Force application quit with a short delay from PyQt6.QtCore import QTimer + QTimer.singleShot(100, lambda: QApplication.instance().quit()) # ========== THEME MANAGEMENT METHODS ========== @@ -1337,10 +1545,7 @@ def switch_to_light_theme(self): def load_theme_from_file(self): """Load theme from JSON configuration file.""" file_path, _ = QFileDialog.getOpenFileName( - self, - "Load Theme Configuration", - "", - "JSON Files (*.json);;All Files (*)" + self, "Load Theme Configuration", "", "JSON Files (*.json);;All Files (*)" ) if file_path: @@ -1351,7 +1556,7 @@ def load_theme_from_file(self): QMessageBox.warning( self, "Theme Load Error", - f"Failed to load theme from {Path(file_path).name}" + f"Failed to load theme from {Path(file_path).name}", ) def save_theme_to_file(self): @@ -1360,7 +1565,7 @@ def save_theme_to_file(self): self, "Save Theme Configuration", "pyqt6_color_scheme.json", - "JSON Files (*.json);;All Files (*)" + "JSON Files (*.json);;All Files (*)", ) if file_path: @@ -1371,24 +1576,24 @@ def save_theme_to_file(self): QMessageBox.warning( self, "Theme Save Error", - f"Failed to save theme to {Path(file_path).name}" + f"Failed to save theme to {Path(file_path).name}", ) def _on_plate_progress_started(self, max_value: int): """Handle plate manager progress started signal.""" - if hasattr(self, '_status_progress_bar'): + if hasattr(self, "_status_progress_bar"): self._status_progress_bar.setMaximum(max_value) self._status_progress_bar.setValue(0) self._status_progress_bar.setVisible(True) def _on_plate_progress_updated(self, value: int): """Handle plate manager progress updated signal.""" - if hasattr(self, '_status_progress_bar'): + if hasattr(self, "_status_progress_bar"): self._status_progress_bar.setValue(value) def _on_plate_progress_finished(self): """Handle plate manager progress finished signal.""" - if hasattr(self, '_status_progress_bar'): + if hasattr(self, "_status_progress_bar"): self._status_progress_bar.setVisible(False) def _on_create_custom_function(self): @@ -1405,7 +1610,7 @@ def _on_create_custom_function(self): parent=self, initial_content=template, title="Create Custom Function", - code_type='function' + code_type="function", ) if editor.exec(): @@ -1419,19 +1624,22 @@ def _on_create_custom_function(self): QMessageBox.information( self, "Success", - f"Function(s) '{func_names}' registered successfully!" + f"Function(s) '{func_names}' registered successfully!", ) except ValidationError as e: # Validation failed - show specific error QMessageBox.critical( self, "Validation Failed", - f"Function code validation failed:\n\n{str(e)}" + f"Function code validation failed:\n\n{str(e)}", ) # Let other exceptions propagate (fail-loud) def _on_manage_custom_functions(self): """Open custom function manager dialog.""" - from openhcs.pyqt_gui.dialogs.custom_function_manager_dialog import CustomFunctionManagerDialog + from openhcs.pyqt_gui.dialogs.custom_function_manager_dialog import ( + CustomFunctionManagerDialog, + ) + dialog = CustomFunctionManagerDialog(parent=self) dialog.exec() diff --git a/openhcs/pyqt_gui/services/llm_pipeline_service.py b/openhcs/pyqt_gui/services/llm_pipeline_service.py index 622503a63..1b7df6f96 100644 --- a/openhcs/pyqt_gui/services/llm_pipeline_service.py +++ b/openhcs/pyqt_gui/services/llm_pipeline_service.py @@ -24,12 +24,12 @@ # Preferred models in priority order (first available wins) PREFERRED_MODELS = [ - "qwen2.5-coder", # Best for code generation - "codellama", # Good alternative - "deepseek-coder", # Another good option - "llama3", # General purpose fallback - "llama2", # Older but common - "mistral", # General purpose + "qwen2.5-coder", # Best for code generation + "codellama", # Good alternative + "deepseek-coder", # Another good option + "llama3", # General purpose fallback + "llama2", # Older but common + "mistral", # General purpose ] # Functions that get full documentation in system prompt (most commonly used) @@ -58,8 +58,9 @@ class LLMPipelineService: containing OpenHCS API documentation and examples. """ - def __init__(self, api_endpoint: str = DEFAULT_OLLAMA_ENDPOINT, - model: str = None): + def __init__( + self, api_endpoint: str = DEFAULT_OLLAMA_ENDPOINT, model: Optional[str] = None + ): """ Initialize LLM service. @@ -72,14 +73,20 @@ def __init__(self, api_endpoint: str = DEFAULT_OLLAMA_ENDPOINT, self.model = model # May be None, resolved on first test_connection # Build system prompts for different contexts self._system_prompts = { - 'pipeline': self._build_pipeline_system_prompt(), - 'function': self._build_custom_function_system_prompt(), + "pipeline": self._build_pipeline_system_prompt(), + "function": self._build_custom_function_system_prompt(), } @property def system_prompt(self) -> str: """Default system prompt (pipeline) for backward compatibility.""" - return self._system_prompts.get('pipeline', '') + return self._system_prompts.get("pipeline", "") + + def get_system_prompt(self, code_type: str = "pipeline") -> str: + """Return the runtime-generated system prompt for a given context.""" + if code_type == "function": + return self._system_prompts.get("function", self.system_prompt) + return self._system_prompts.get("pipeline", self.system_prompt) def _derive_base_url(self, endpoint: str) -> str: """Extract base URL from endpoint.""" @@ -90,8 +97,7 @@ def _get_available_models(self) -> List[str]: """Fetch available models from Ollama.""" try: response = requests.get( - f"{self.base_url}/api/tags", - timeout=CONNECTION_TIMEOUT_S + f"{self.base_url}/api/tags", timeout=CONNECTION_TIMEOUT_S ) response.raise_for_status() data = response.json() @@ -108,7 +114,9 @@ def _select_best_model(self, available_models: List[str]) -> Optional[str]: for preferred in PREFERRED_MODELS: for available in available_models: # Match base name (e.g., "qwen2.5-coder" matches "qwen2.5-coder:7b") - if available.split(":")[0] == preferred or available.startswith(preferred): + if available.split(":")[0] == preferred or available.startswith( + preferred + ): return available # Fall back to first available model @@ -261,10 +269,12 @@ def _build_custom_function_system_prompt(self) -> str: === CRITICAL RULES === 1. Include ALL imports (dataclass, typing, numpy, etc.) -2. First parameter MUST be named 'image' (3D array: channels, height, width) -3. Return a 3D array, OR tuple (image, output1, output2, ...) with @special_outputs +2. First parameter MUST be named 'image' (3D array: (C, Y, X) a.k.a. (Z, Y, X)) +3. Input/Output backend types are declared by the memory decorator (e.g. @numpy / @cupy / @pyclesperanto). Your function must accept the decorator's declared input type and return the declared output type. 4. DO NOT write FunctionStep or pipeline code - just the function 5. Output ONLY Python code, no explanations +6. Do NOT manually convert between array backends inside the function (no cp.asnumpy(), no cle.pull(), etc.). OpenHCS handles cross-step conversions. +7. The decorator adds keyword-only args like slice_by_slice and dtype_conversion. dtype_conversion defaults to preserving the input dtype for the main output. {imports_section} @@ -282,7 +292,7 @@ def enhance_contrast(image, clip_limit: float = 0.03): ``` === FUNCTION WITH CSV OUTPUT === -When you need to save measurements to CSV, use @special_outputs with csv_materializer. +When you need to save measurements to CSV, use @special_outputs with csv_only() preset. RETURN SEMANTICS: With N special_outputs, return (image, output1, output2, ..., outputN) @@ -293,7 +303,7 @@ def enhance_contrast(image, clip_limit: float = 0.03): from skimage.measure import label, regionprops from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import csv_materializer +from openhcs.processing.materialization import csv_only @dataclass class CellMeasurement: @@ -303,10 +313,7 @@ class CellMeasurement: mean_intensity: float @numpy -@special_outputs(("cell_measurements", csv_materializer( - fields=["slice_index", "cell_count", "total_area", "mean_intensity"], - analysis_type="cell_counts" -))) +@special_outputs(("cell_measurements", csv_only())) def count_cells_with_csv( image, threshold: float = 0.5, @@ -339,10 +346,10 @@ def count_cells_with_csv( from skimage.measure import label from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import roi_zip_materializer +from openhcs.processing.materialization import roi_zip @numpy -@special_outputs(("segmentation_masks", roi_zip_materializer())) +@special_outputs(("segmentation_masks", roi_zip())) def segment_cells_with_rois( image, threshold: float = 0.5 @@ -357,33 +364,27 @@ def segment_cells_with_rois( return image, masks # masks -> .roi.zip for Fiji, shapes for Napari ``` -=== FUNCTION WITH BOTH CSV AND ROI OUTPUT === +=== FUNCTION WITH BOTH JSON AND CSV OUTPUT === +Use json_and_csv() preset for analysis results (most common pattern). + ```python from typing import List, Tuple import numpy as np from skimage.measure import label from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import regionprops_materializer +from openhcs.processing.materialization import json_and_csv, roi_zip @numpy @special_outputs( - # One special output can materialize multiple artifacts: - # - ROI zip for Fiji + shapes for Napari - # - CSV details table - # - JSON summary - ("segmentation_masks", regionprops_materializer( - analysis_type="segmentation_regionprops", - min_area=50, - require_intensity=True, - intensity_source="step_output", - )) + ("segmentation_masks", roi_zip()), + ("cell_measurements", json_and_csv()), ) def analyze_cells_full( image, threshold: float = 0.5 ) -> Tuple[np.ndarray, List[np.ndarray]]: - """Segmentation + rich materialization (ROIs + metrics) from labeled masks.""" + """Segmentation + analysis with both ROIs and metrics output.""" masks: List[np.ndarray] = [] for i, slice_2d in enumerate(image): @@ -391,7 +392,6 @@ def analyze_cells_full( labeled = label(binary) masks.append(labeled) - # 1 special_output -> return (image, masks) return image, masks ``` @@ -407,8 +407,7 @@ def analyze_cells_full( import pyclesperanto as cle from openhcs.core.memory import pyclesperanto from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import csv_materializer -from openhcs.processing.materialization import roi_zip_materializer +from openhcs.processing.materialization import csv_only, roi_zip @dataclass class CellStats: @@ -419,8 +418,8 @@ class CellStats: @pyclesperanto @special_outputs( - ("cell_stats", csv_materializer(fields=["slice_index", "cell_count", "total_area", "mean_intensity"], analysis_type="cell_stats")), - ("segmentation_masks", roi_zip_materializer()) + ("cell_stats", csv_only()), + ("segmentation_masks", roi_zip()) ) def count_cells_gpu( image, @@ -455,9 +454,9 @@ def count_cells_gpu( total_area=total_area, mean_intensity=mean_int )) - masks.append(cle.pull(labels)) # Pull labeled mask for ROI output + masks.append(labels) - return cle.pull(image), stats_list, masks + return image, stats_list, masks ``` Key pyclesperanto functions: @@ -480,8 +479,7 @@ def count_cells_gpu( from cucim.skimage.measure import label, regionprops_table from openhcs.core.memory import cupy from openhcs.core.pipeline.function_contracts import special_outputs -from openhcs.processing.materialization import csv_materializer -from openhcs.processing.materialization import roi_zip_materializer +from openhcs.processing.materialization import CsvOptions, MaterializationSpec, ROIOptions @dataclass class CellStats: @@ -492,8 +490,8 @@ class CellStats: @cupy @special_outputs( - ("cell_stats", csv_materializer(fields=["slice_index", "cell_count", "total_area", "mean_intensity"], analysis_type="cell_stats")), - ("segmentation_masks", roi_zip_materializer()) + ("cell_stats", MaterializationSpec(CsvOptions(filename_suffix="_stats.csv"))), + ("segmentation_masks", MaterializationSpec(ROIOptions())) ) def count_cells_cupy( image, @@ -525,9 +523,9 @@ def count_cells_cupy( total_area=float(cp.sum(areas[valid_mask])), mean_intensity=float(cp.mean(props['mean_intensity'][valid_mask])) if cp.any(valid_mask) else 0.0 )) - masks.append(cp.asnumpy(labeled)) # Convert to numpy for ROI output + masks.append(labeled) - return cp.asnumpy(image), stats_list, masks + return image, stats_list, masks ``` Key cucim.skimage modules (same API as skimage, but GPU): @@ -537,7 +535,7 @@ def count_cells_cupy( - cucim.skimage.segmentation: watershed, clear_border - cucim.skimage.exposure: equalize_adapthist, rescale_intensity -IMPORTANT: Use cp.asnumpy() to convert cupy arrays to numpy for output. +IMPORTANT: Do not convert arrays between backends on return. === SPECIAL INPUTS (consume data from previous steps) === ```python @@ -556,7 +554,7 @@ def analyze_at_positions(image, cell_positions): def _get_dynamic_imports_section(self) -> str: """Generate imports section with actual module paths.""" # These are the actual import paths - single source of truth - return '''=== REQUIRED IMPORTS (use exactly these paths) === + return """=== REQUIRED IMPORTS (use exactly these paths) === # Backend decorators from openhcs.core.memory import numpy, pyclesperanto, cupy @@ -565,51 +563,104 @@ def _get_dynamic_imports_section(self) -> str: # Materializers for CSV/JSON and ROI outputs from openhcs.processing.materialization import ( - csv_materializer, - json_materializer, - dual_materializer, - roi_zip_materializer + MaterializationSpec, + CsvOptions, + JsonOptions, + ROIOptions, + TiffStackOptions, + TextOptions, ) # Standard library (include as needed) from dataclasses import dataclass from typing import List, Tuple, Optional -import numpy as np''' +import numpy as np""" def _get_dynamic_materializers_section(self) -> str: - """Generate materializers section with signatures from actual code.""" + """Generate materializers section with simple presets and advanced options.""" try: - from openhcs.processing.materialization import csv_materializer, json_materializer, dual_materializer, roi_zip_materializer + from openhcs.processing.materialization import ( + MaterializationSpec, + CsvOptions, + JsonOptions, + ROIOptions, + TiffStackOptions, + TextOptions, + ) + from openhcs.processing.materialization.presets import ( + json_and_csv, + csv_only, + json_only, + roi_zip, + tiff_stack, + text_only, + ) import inspect - csv_sig = str(inspect.signature(csv_materializer)) - json_sig = str(inspect.signature(json_materializer)) - dual_sig = str(inspect.signature(dual_materializer)) - roi_sig = str(inspect.signature(roi_zip_materializer)) + csv_sig = str(inspect.signature(CsvOptions)) + json_sig = str(inspect.signature(JsonOptions)) + roi_sig = str(inspect.signature(ROIOptions)) + + return f"""=== SIMPLE MATERIALIZATION (Use These!) === +from openhcs.processing.materialization import json_and_csv, csv_only, json_only, roi_zip + +# JSON + CSV (most common for analysis) +@special_outputs(("results", json_and_csv())) + +# CSV only +@special_outputs(("measurements", csv_only())) + +# JSON only +@special_outputs(("metadata", json_only())) + +# ROI zip for ImageJ/Fiji +@special_outputs(("masks", roi_zip())) + +=== ADVANCED CUSTOMIZATION (When needed) === +CsvOptions{csv_sig} +JsonOptions{json_sig} +ROIOptions{roi_sig} - return f'''=== MATERIALIZER SIGNATURES === -csv_materializer{csv_sig} -json_materializer{json_sig} -dual_materializer{dual_sig} -roi_zip_materializer{roi_sig}''' +Usage: MaterializationSpec(CsvOptions(filename_suffix="_custom.csv", fields=["x", "y"]))""" except Exception: - return '''=== MATERIALIZERS === -csv_materializer(fields: List[str], analysis_type: str, filename_suffix: str = ".csv", strip_roi_suffix: bool = False) -json_materializer(fields: List[str], analysis_type: str, filename_suffix: str = ".json", strip_roi_suffix: bool = False) -dual_materializer(fields: List[str], summary_fields: List[str], analysis_type: str, strip_roi_suffix: bool = False) -roi_zip_materializer(min_area: int = 10, extract_contours: bool = True, strip_roi_suffix: bool = False)''' + return """=== SIMPLE MATERIALIZATION (Use These!) === +from openhcs.processing.materialization import json_and_csv, csv_only, json_only, roi_zip + +# Most common patterns - just use these: +@special_outputs(("results", json_and_csv())) # JSON + CSV +@special_outputs(("measurements", csv_only())) # CSV only +@special_outputs(("masks", roi_zip())) # ROIs for ImageJ + +=== ADVANCED CUSTOMIZATION === +MaterializationSpec(CsvOptions(...), JsonOptions(...))""" def _get_pyclesperanto_function_docs(self) -> str: """Get pyclesperanto functions dynamically if available.""" try: - import pyclesperanto as cle + import pyclesperanto as cle # type: ignore[import-not-found] + # Get commonly used functions that actually exist key_funcs = [] - for name in ["gaussian_blur", "median", "top_hat", "bottom_hat", - "threshold_otsu", "binary_opening", "binary_closing", - "erode", "dilate", "label", "connected_components_labeling", - "voronoi_labeling", "exclude_labels_on_edges", "exclude_small_labels", - "push", "pull", "maximum_z_projection", "mean_z_projection"]: + for name in [ + "gaussian_blur", + "median", + "top_hat", + "bottom_hat", + "threshold_otsu", + "binary_opening", + "binary_closing", + "erode", + "dilate", + "label", + "connected_components_labeling", + "voronoi_labeling", + "exclude_labels_on_edges", + "exclude_small_labels", + "push", + "pull", + "maximum_z_projection", + "mean_z_projection", + ]: if hasattr(cle, name): key_funcs.append(f"cle.{name}()") return "\n".join(key_funcs) @@ -619,7 +670,10 @@ def _get_pyclesperanto_function_docs(self) -> str: def _get_function_documentation(self) -> str: """Build function documentation from the registry.""" try: - from openhcs.processing.backends.lib_registry.registry_service import RegistryService + from openhcs.processing.backends.lib_registry.registry_service import ( + RegistryService, + ) + all_functions = RegistryService.get_all_functions_with_metadata() except Exception as e: logger.warning(f"Could not load function registry: {e}") @@ -646,7 +700,10 @@ def _get_function_documentation(self) -> str: # Sort with core functions first, then alphabetically sorted_functions = sorted( functions, - key=lambda m: (0 if m.original_name in CORE_FUNCTIONS else 1, m.original_name) + key=lambda m: ( + 0 if m.original_name in CORE_FUNCTIONS else 1, + m.original_name, + ), ) # Limit non-core functions to keep prompt size manageable @@ -663,7 +720,9 @@ def _get_function_documentation(self) -> str: count += 1 if len(sorted_functions) > MAX_FUNCTIONS_PER_LIBRARY: - lib_docs.append(f"... and {len(sorted_functions) - MAX_FUNCTIONS_PER_LIBRARY} more functions\n") + lib_docs.append( + f"... and {len(sorted_functions) - MAX_FUNCTIONS_PER_LIBRARY} more functions\n" + ) docs_parts.append("\n".join(lib_docs)) @@ -690,7 +749,7 @@ def _format_function_doc(self, metadata) -> str: # Get description (first line of docstring) desc = "" if metadata.doc: - first_line = metadata.doc.split('\n')[0].strip() + first_line = metadata.doc.split("\n")[0].strip() if first_line: desc = f" # {first_line}" @@ -721,7 +780,10 @@ def _format_signature(self, func: Callable, name: str) -> str: if pname in INTERNAL_PARAMS or pname == "kwargs": continue # Skip *args, **kwargs - if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + if param.kind in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ): continue if param.default is inspect.Parameter.empty: @@ -752,7 +814,10 @@ def _format_parameters(self, func: Callable) -> str: for pname, param in sig.parameters.items(): if pname in INTERNAL_PARAMS: continue - if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + if param.kind in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ): continue # Get type annotation @@ -788,20 +853,31 @@ def _get_enum_documentation(self) -> str: # Core enums that are always useful try: from openhcs.processing.backends.analysis.cell_counting_cpu import ( - DetectionMethod, ThresholdMethod, ColocalizationMethod + DetectionMethod, + ThresholdMethod, + ColocalizationMethod, + ) + + enums_to_document.extend( + [DetectionMethod, ThresholdMethod, ColocalizationMethod] ) - enums_to_document.extend([DetectionMethod, ThresholdMethod, ColocalizationMethod]) except ImportError: pass try: - from openhcs.constants.constants import DtypeConversion, VariableComponents, GroupBy + from openhcs.constants.constants import ( + DtypeConversion, + VariableComponents, + GroupBy, + ) + enums_to_document.extend([DtypeConversion, VariableComponents, GroupBy]) except ImportError: pass try: from openhcs.constants.input_source import InputSource + enums_to_document.append(InputSource) except ImportError: pass @@ -816,9 +892,11 @@ def _get_enum_documentation(self) -> str: def _get_example_pipeline(self) -> str: """Load example pipeline from file.""" - basic_pipeline_path = Path(__file__).parent.parent.parent / "tests" / "basic_pipeline.py" + basic_pipeline_path = ( + Path(__file__).parent.parent.parent / "tests" / "basic_pipeline.py" + ) try: - with open(basic_pipeline_path, 'r') as f: + with open(basic_pipeline_path, "r") as f: return f.read() except Exception as e: logger.warning(f"Could not load example pipeline: {e}") @@ -835,7 +913,7 @@ def _get_fallback_function_docs(self) -> str: - `assemble_stack_cpu`: from openhcs.processing.backends.assemblers.assemble_stack_cpu """ - def generate_code(self, user_request: str, code_type: str = 'pipeline') -> str: + def generate_code(self, user_request: str, code_type: str = "pipeline") -> str: """ Generate code from user request based on context. @@ -851,16 +929,18 @@ def generate_code(self, user_request: str, code_type: str = 'pipeline') -> str: """ try: # Select appropriate system prompt based on code_type - if code_type == 'function': - system_prompt = self._system_prompts.get('function', self.system_prompt) - context_suffix = "Generate a standalone custom function with @decorator." + if code_type == "function": + system_prompt = self._system_prompts.get("function", self.system_prompt) + context_suffix = ( + "Generate a standalone custom function with @decorator." + ) else: - system_prompt = self._system_prompts.get('pipeline', self.system_prompt) + system_prompt = self._system_prompts.get("pipeline", self.system_prompt) context_suffix = { - 'pipeline': "Generate complete pipeline_steps list with FunctionStep objects.", - 'step': "Generate a single FunctionStep object.", - 'config': "Generate a configuration object (LazyProcessingConfig, LazyStepWellFilterConfig, etc.).", - 'orchestrator': "Generate complete orchestrator code with plate_paths, pipeline_data, and configs." + "pipeline": "Generate complete pipeline_steps list with FunctionStep objects.", + "step": "Generate a single FunctionStep object.", + "config": "Generate a configuration object (LazyProcessingConfig, LazyStepWellFilterConfig, etc.).", + "orchestrator": "Generate complete orchestrator code with plate_paths, pipeline_data, and configs.", }.get(code_type, "Generate OpenHCS code.") # Construct request payload (Ollama format) @@ -871,15 +951,17 @@ def generate_code(self, user_request: str, code_type: str = 'pipeline') -> str: "options": { "temperature": 0.2, # Low temperature for more deterministic code generation "top_p": 0.9, - } + }, } - logger.info(f"Sending request to LLM: {self.api_endpoint} (code_type={code_type})") + logger.info( + f"Sending request to LLM: {self.api_endpoint} (code_type={code_type})" + ) response = requests.post(self.api_endpoint, json=payload, timeout=60) response.raise_for_status() result = response.json() - generated_code = result.get('response', '') + generated_code = result.get("response", "") # Clean up code (remove markdown code blocks if present) generated_code = self._clean_generated_code(generated_code) @@ -906,7 +988,7 @@ def _clean_generated_code(self, code: str) -> str: """ # Remove markdown code blocks if code.startswith("```python"): - code = code[len("```python"):].lstrip() + code = code[len("```python") :].lstrip() if code.startswith("```"): code = code[3:].lstrip() if code.endswith("```"): diff --git a/openhcs/pyqt_gui/services/reactor_providers.py b/openhcs/pyqt_gui/services/reactor_providers.py index 629ed27b6..54bbbb47e 100644 --- a/openhcs/pyqt_gui/services/reactor_providers.py +++ b/openhcs/pyqt_gui/services/reactor_providers.py @@ -48,8 +48,15 @@ class OpenHCSFormGenConfig(FormGenConfig): class OpenHCSCodegenProvider: """Codegen provider backed by pycodify with OpenHCS formatters.""" - def generate_complete_orchestrator_code(self, plate_paths, pipeline_data, global_config=None, - per_plate_configs=None, pipeline_config=None, clean_mode=True) -> str: + def generate_complete_orchestrator_code( + self, + plate_paths, + pipeline_data, + global_config=None, + per_plate_configs=None, + pipeline_config=None, + clean_mode=True, + ) -> str: from openhcs.core.config import PipelineConfig from pycodify import Assignment, BlankLine, CodeBlock, generate_python_source @@ -78,7 +85,9 @@ def generate_complete_orchestrator_code(self, plate_paths, pipeline_data, global clean_mode=clean_mode, ) - def generate_complete_pipeline_steps_code(self, pipeline_steps, clean_mode=True) -> str: + def generate_complete_pipeline_steps_code( + self, pipeline_steps, clean_mode=True + ) -> str: from pycodify import Assignment, generate_python_source return generate_python_source( @@ -87,7 +96,9 @@ def generate_complete_pipeline_steps_code(self, pipeline_steps, clean_mode=True) clean_mode=clean_mode, ) - def generate_complete_function_pattern_code(self, func_obj, clean_mode=False) -> str: + def generate_complete_function_pattern_code( + self, func_obj, clean_mode=False + ) -> str: from pycodify import Assignment, generate_python_source return generate_python_source( @@ -105,7 +116,9 @@ def generate_step_code(self, step_obj, clean_mode=True) -> str: clean_mode=clean_mode, ) - def generate_config_code(self, config_obj, clean_mode=True, config_class: Optional[type] = None) -> str: + def generate_config_code( + self, config_obj, clean_mode=True, config_class: Optional[type] = None + ) -> str: from pycodify import Assignment, generate_python_source return generate_python_source( @@ -119,7 +132,10 @@ class OpenHCSFunctionRegistry: """Adapter for OpenHCS RegistryService.""" def _get_metadata(self) -> Dict[str, Any]: - from openhcs.processing.backends.lib_registry.registry_service import RegistryService + from openhcs.processing.backends.lib_registry.registry_service import ( + RegistryService, + ) + return RegistryService.get_all_functions_with_metadata() def get_function_by_name(self, name: str) -> Optional[Callable]: @@ -161,18 +177,31 @@ class OpenHCSLogDiscoveryProvider: def get_current_log_path(self) -> Path: from openhcs.core.log_utils import get_current_log_file_path + return Path(get_current_log_file_path()) - def discover_logs(self, base_log_path: Optional[str] = None, include_main_log: bool = True, - log_directory: Optional[Path] = None): + def discover_logs( + self, + base_log_path: Optional[str] = None, + include_main_log: bool = True, + log_directory: Optional[Path] = None, + ): from openhcs.core.log_utils import discover_logs from pyqt_reactive.core.log_utils import LogFileInfo - logs = discover_logs(base_log_path=base_log_path, include_main_log=include_main_log, - log_directory=log_directory) + logs = discover_logs( + base_log_path=base_log_path, + include_main_log=include_main_log, + log_directory=log_directory, + ) # Convert to pyqt_reactive LogFileInfo for consistency converted = [ - LogFileInfo(path=l.path, log_type=l.log_type, worker_id=l.worker_id, display_name=l.display_name) + LogFileInfo( + path=l.path, + log_type=l.log_type, + worker_id=l.worker_id, + display_name=l.display_name, + ) for l in logs ] return converted @@ -189,7 +218,10 @@ def scan_for_server_logs(self): from openhcs.core.config import get_all_streaming_ports from openhcs.constants.constants import CONTROL_PORT_OFFSET from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG - from zmqruntime.transport import get_zmq_transport_url, get_default_transport_mode + from zmqruntime.transport import ( + get_zmq_transport_url, + get_default_transport_mode, + ) from openhcs.core.log_utils import classify_log_file from pyqt_reactive.core.log_utils import LogFileInfo @@ -213,7 +245,7 @@ def ping_server(port: int) -> dict: ) socket.connect(control_url) - socket.send(pickle.dumps({'type': 'ping'})) + socket.send(pickle.dumps({"type": "ping"})) response = socket.recv() pong = pickle.loads(response) @@ -225,13 +257,17 @@ def ping_server(port: int) -> dict: for port in ports_to_scan: pong = ping_server(port) - if pong and pong.get('log_file_path'): - log_path = Path(pong['log_file_path']) + if pong and pong.get("log_file_path"): + log_path = Path(pong["log_file_path"]) if log_path.exists(): log_info = classify_log_file(log_path, None, False) discovered.append( - LogFileInfo(path=log_info.path, log_type=log_info.log_type, - worker_id=log_info.worker_id, display_name=log_info.display_name) + LogFileInfo( + path=log_info.path, + log_type=log_info.log_type, + worker_id=log_info.worker_id, + display_name=log_info.display_name, + ) ) return discovered @@ -241,45 +277,130 @@ class OpenHCSComponentSelectionProvider: def get_groupby_enum(self) -> Any: from openhcs.constants.constants import GroupBy + return GroupBy def _get_plate_manager(self): - from PyQt6.QtWidgets import QApplication from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + from pyqt_reactive.services import ServiceRegistry - for widget in QApplication.topLevelWidgets(): - if hasattr(widget, 'floating_windows'): - plate_dialog = widget.floating_windows.get("plate_manager") - if plate_dialog: - return plate_dialog.findChild(PlateManagerWidget) - return None + return ServiceRegistry.get(PlateManagerWidget) def _get_current_orchestrator(self): + import logging + + logger = logging.getLogger(__name__) + plate_manager = self._get_plate_manager() + debug_info = f"[_get_current_orchestrator] plate_manager={plate_manager}\n" if not plate_manager: + debug_info += "[_get_current_orchestrator] No plate manager found!\n" + # Store debug info for exception + self._last_debug_info = debug_info return None + + current_plate = plate_manager.selected_plate_path + debug_info += f"[_get_current_orchestrator] current_plate={current_plate!r}\n" + + from objectstate import ObjectStateRegistry + from openhcs.core.orchestrator.orchestrator import OrchestratorState + + orchestrator = ObjectStateRegistry.get_object(current_plate) + debug_info += f"[_get_current_orchestrator] orchestrator={orchestrator}\n" + + if orchestrator: + debug_info += ( + f"[_get_current_orchestrator] orchestrator.state={orchestrator.state}\n" + ) + debug_info += f"[_get_current_orchestrator] OrchestratorState.CREATED={OrchestratorState.CREATED}\n" + debug_info += f"[_get_current_orchestrator] state != CREATED: {orchestrator.state != OrchestratorState.CREATED}\n" + + # Use same check as plate manager - check state != CREATED + if orchestrator and orchestrator.state != OrchestratorState.CREATED: + debug_info += f"[_get_current_orchestrator] Returning orchestrator\n" + self._last_debug_info = debug_info + return orchestrator + + debug_info += ( + f"[_get_current_orchestrator] Returning None! orchestrator={orchestrator}\n" + ) + self._last_debug_info = debug_info + return None + current_plate = plate_manager.selected_plate_path + logger.info(f"[_get_current_orchestrator] current_plate={current_plate!r}") + from objectstate import ObjectStateRegistry + from openhcs.core.orchestrator.orchestrator import OrchestratorState + orchestrator = ObjectStateRegistry.get_object(current_plate) - if orchestrator and orchestrator.is_initialized(): + logger.info(f"[_get_current_orchestrator] orchestrator={orchestrator}") + + if orchestrator: + logger.info( + f"[_get_current_orchestrator] orchestrator.state={orchestrator.state}" + ) + logger.info( + f"[_get_current_orchestrator] OrchestratorState.CREATED={OrchestratorState.CREATED}" + ) + logger.info( + f"[_get_current_orchestrator] state != CREATED: {orchestrator.state != OrchestratorState.CREATED}" + ) + + # Use same check as plate manager - check state != CREATED + if orchestrator and orchestrator.state != OrchestratorState.CREATED: + logger.info(f"[_get_current_orchestrator] Returning orchestrator") return orchestrator + + logger.error( + f"[_get_current_orchestrator] Returning None! orchestrator={orchestrator}" + ) return None + def has_components_available(self, group_by: Any) -> bool: + """Check if components are available without fetching them all.""" + orchestrator = self._get_current_orchestrator() + if not orchestrator: + return False + # Try to get component keys - if empty, no components available + try: + components = orchestrator.get_component_keys(group_by) + return len(components) > 0 + except Exception: + return False + def get_component_keys(self, group_by: Any) -> List[str]: orchestrator = self._get_current_orchestrator() if not orchestrator: - return [] + # Return debug info as the error so it shows in the UI + debug_info = getattr(self, "_last_debug_info", "No debug info available") + raise RuntimeError( + f"Cannot get component keys - no initialized orchestrator found!\n\n" + f"DEBUG INFO:\n{debug_info}\n" + f"GROUP_BY: {group_by}" + ) return orchestrator.get_component_keys(group_by) - def get_component_display_name(self, group_by: Any, component_key: str) -> Optional[str]: + def get_component_display_name( + self, group_by: Any, component_key: str + ) -> Optional[str]: orchestrator = self._get_current_orchestrator() if not orchestrator: return None - return orchestrator.metadata_cache.get_component_metadata(group_by, component_key) + return orchestrator.metadata_cache.get_component_metadata( + group_by, component_key + ) - def select_components(self, available_components: Iterable[str], selected_components: Iterable[str], - group_by: Any, parent: Optional[Any] = None, **context: Any) -> Optional[List[str]]: + def select_components( + self, + available_components: Iterable[str], + selected_components: Iterable[str], + group_by: Any, + parent: Optional[Any] = None, + **context: Any, + ) -> Optional[List[str]]: from pyqt_reactive.dialogs.group_by_selector_dialog import GroupBySelectorDialog + return GroupBySelectorDialog.select_components( available_components=list(available_components), selected_components=list(selected_components), @@ -292,130 +413,27 @@ def select_components(self, available_components: Iterable[str], selected_compon class OpenHCSFunctionSelectionProvider: """Function selection provider backed by OpenHCS dialogs.""" - def select_function(self, parent: Optional[Any] = None, **context: Any) -> Optional[Callable]: - from openhcs.pyqt_gui.dialogs.function_selector_dialog import FunctionSelectorDialog - return FunctionSelectorDialog.select_function(parent=parent) - - -class OpenHCSWindowFactory: - """Window factory for OpenHCS scope navigation.""" - - def create_window_for_scope(self, scope_id: str, object_state: Optional[Any] = None) -> Optional[QWidget]: - if scope_id == "": - return self._create_global_config_window() - if "::" not in scope_id: - return self._create_plate_config_window(scope_id) - return self._create_step_editor_window(scope_id, object_state) - - def _create_global_config_window(self) -> Optional[QWidget]: - from openhcs.pyqt_gui.windows.config_window import ConfigWindow - from openhcs.core.config import GlobalPipelineConfig - from openhcs.config_framework.global_config import get_current_global_config, set_global_config_for_editing - - current_config = get_current_global_config(GlobalPipelineConfig) or GlobalPipelineConfig() - - def handle_save(new_config): - set_global_config_for_editing(GlobalPipelineConfig, new_config) - - window = ConfigWindow( - config_class=GlobalPipelineConfig, - current_config=current_config, - on_save_callback=handle_save, - scope_id="", + def select_function( + self, parent: Optional[Any] = None, **context: Any + ) -> Optional[Callable]: + from openhcs.pyqt_gui.dialogs.function_selector_dialog import ( + FunctionSelectorDialog, ) - window.show(); window.raise_(); window.activateWindow() - return window - - def _create_plate_config_window(self, scope_id: str) -> Optional[QWidget]: - from openhcs.pyqt_gui.windows.config_window import ConfigWindow - from openhcs.core.config import PipelineConfig - orchestrator = self._get_orchestrator(scope_id) - if not orchestrator: - return None - window = ConfigWindow( - config_class=PipelineConfig, - current_config=orchestrator.pipeline_config, - on_save_callback=None, - scope_id=scope_id, - ) - window.show(); window.raise_(); window.activateWindow() - return window - - def _create_step_editor_window(self, scope_id: str, object_state: Optional[Any] = None) -> Optional[QWidget]: - from openhcs.pyqt_gui.windows.dual_editor_window import DualEditorWindow - from objectstate import ObjectStateRegistry - - parts = scope_id.split("::") - if len(parts) < 2: - return None - - plate_path = parts[0] - step_token = parts[1] - is_function_scope = len(parts) >= 3 - step_scope_id = f"{parts[0]}::{parts[1]}" - - orchestrator = self._get_orchestrator(plate_path) - if not orchestrator: - return None - - step = None - if object_state: - if is_function_scope: - step_state = ObjectStateRegistry.get_by_scope(step_scope_id) - step = step_state.object_instance if step_state else None - else: - step = object_state.object_instance - if not step: - step = self._find_step_by_token(plate_path, step_token) - - if not step: - return None - - window = DualEditorWindow( - step_data=step, - is_new=False, - on_save_callback=None, - orchestrator=orchestrator, - parent=None, - ) - if is_function_scope and window.tab_widget: - window.tab_widget.setCurrentIndex(1) - - window.show(); window.raise_(); window.activateWindow() - return window + return FunctionSelectorDialog.select_function(parent=parent) - def _get_orchestrator(self, plate_path: str): - from objectstate import ObjectStateRegistry - return ObjectStateRegistry.get_object(plate_path) - def _get_plate_manager(self): - from PyQt6.QtWidgets import QApplication - from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget +def register_openhcs_window_handlers(): + """Register OpenHCS window handlers with the generic factory. - for widget in QApplication.topLevelWidgets(): - if hasattr(widget, 'floating_windows'): - plate_dialog = widget.floating_windows.get("plate_manager") - if plate_dialog: - return plate_dialog.findChild(PlateManagerWidget) - return None + Note: This is a convenience re-export. The actual registration + happens in window_handlers module. + """ + from openhcs.pyqt_gui.services.window_handlers import ( + register_openhcs_window_handlers as _register, + ) - def _find_step_by_token(self, plate_path: str, step_token: str): - from objectstate import ObjectStateRegistry - pipeline_scope = f"{plate_path}::pipeline" - pipeline_state = ObjectStateRegistry.get_by_scope(pipeline_scope) - if not pipeline_state: - return None - step_scope_ids = pipeline_state.parameters.get("step_scope_ids") or [] - for scope_id in step_scope_ids: - step_state = ObjectStateRegistry.get_by_scope(scope_id) - if step_state: - step = step_state.object_instance - if getattr(step, '_scope_token', None) == step_token: - return step - if getattr(step, '_pipeline_scope_token', None) == step_token: - return step - return None + _register() def register_reactor_providers() -> None: @@ -424,6 +442,7 @@ def register_reactor_providers() -> None: config = OpenHCSFormGenConfig() try: from openhcs.core.xdg_paths import get_data_file_path + config.path_cache_file = str(get_data_file_path("path_cache.json")) except Exception: config.path_cache_file = None @@ -440,6 +459,7 @@ def register_reactor_providers() -> None: # Providers from openhcs.pyqt_gui.services.llm_pipeline_service import LLMPipelineService + register_llm_service(LLMPipelineService()) register_codegen_provider(OpenHCSCodegenProvider()) register_function_registry(OpenHCSFunctionRegistry()) @@ -447,12 +467,14 @@ def register_reactor_providers() -> None: register_server_scan_provider(OpenHCSServerScanProvider()) register_component_selection_provider(OpenHCSComponentSelectionProvider()) register_function_selection_provider(OpenHCSFunctionSelectionProvider()) - register_window_factory(OpenHCSWindowFactory()) + # Window handlers are registered in main.py after widgets are created # Preview formatters (OpenHCS-specific) try: from openhcs.core.config import WellFilterConfig - from openhcs.pyqt_gui.widgets.config_preview_formatters import format_config_indicator + from openhcs.pyqt_gui.widgets.config_preview_formatters import ( + format_config_indicator, + ) def _format_config(config_obj, field_name: str) -> Optional[str]: return format_config_indicator(field_name, config_obj) diff --git a/openhcs/pyqt_gui/services/service_adapter.py b/openhcs/pyqt_gui/services/service_adapter.py index 9d9743773..b0b173835 100644 --- a/openhcs/pyqt_gui/services/service_adapter.py +++ b/openhcs/pyqt_gui/services/service_adapter.py @@ -14,7 +14,11 @@ from PyQt6.QtGui import QDesktopServices from PyQt6.QtCore import QUrl -from openhcs.core.path_cache import PathCacheKey, get_cached_dialog_path, cache_dialog_path +from openhcs.core.path_cache import ( + PathCacheKey, + get_cached_dialog_path, + cache_dialog_path, +) from pyqt_reactive.theming import ThemeManager from pyqt_reactive.theming import ColorScheme @@ -50,8 +54,12 @@ class GlobalEventBus(QObject): """ # Global signals that all windows can emit/receive - pipeline_changed = pyqtSignal(list) # List[FunctionStep] - emitted when pipeline changes - config_changed = pyqtSignal(object) # Config object - emitted when any config changes + pipeline_changed = pyqtSignal( + list + ) # List[FunctionStep] - emitted when pipeline changes + config_changed = pyqtSignal( + object + ) # Config object - emitted when any config changes step_changed = pyqtSignal(object) # FunctionStep - emitted when a step is modified def __init__(self): @@ -85,7 +93,9 @@ def emit_pipeline_changed(self, pipeline_steps: list): Args: pipeline_steps: Updated list of FunctionStep objects """ - logger.debug(f"Broadcasting pipeline_changed to {len(self._registered_windows)} windows") + logger.debug( + f"Broadcasting pipeline_changed to {len(self._registered_windows)} windows" + ) self.pipeline_changed.emit(pipeline_steps) def emit_config_changed(self, config): @@ -94,7 +104,9 @@ def emit_config_changed(self, config): Args: config: Updated config object """ - logger.debug(f"Broadcasting config_changed to {len(self._registered_windows)} windows") + logger.debug( + f"Broadcasting config_changed to {len(self._registered_windows)} windows" + ) self.config_changed.emit(config) def emit_step_changed(self, step): @@ -103,18 +115,20 @@ def emit_step_changed(self, step): Args: step: Updated FunctionStep object """ - logger.debug(f"Broadcasting step_changed to {len(self._registered_windows)} windows") + logger.debug( + f"Broadcasting step_changed to {len(self._registered_windows)} windows" + ) self.step_changed.emit(step) class PyQtServiceAdapter: """ Adapter to bridge OpenHCS services to PyQt6 context. - + Replaces prompt_toolkit dependencies (dialogs, system commands, etc.) with PyQt6 equivalents while maintaining the same interface for services. """ - + def __init__(self, main_window: QWidget): """ Initialize the service adapter. @@ -189,7 +203,7 @@ def run_async_in_thread(): raise # Use ThreadPoolExecutor (simpler than Qt threading) - if not hasattr(self, '_thread_pool'): + if not hasattr(self, "_thread_pool"): self._thread_pool = ThreadPoolExecutor(max_workers=4) # Submit to thread pool (non-blocking like TUI executor) @@ -198,27 +212,29 @@ def run_async_in_thread(): def show_dialog(self, content: str, title: str = "OpenHCS") -> bool: """ Replace prompt_toolkit dialogs with QMessageBox. - + Args: content: Dialog content text title: Dialog title - + Returns: True if user clicked OK, False otherwise """ msg = QMessageBox(self.main_window) msg.setWindowTitle(title) msg.setText(content) - msg.setStandardButtons(QMessageBox.StandardButton.Ok | QMessageBox.StandardButton.Cancel) + msg.setStandardButtons( + QMessageBox.StandardButton.Ok | QMessageBox.StandardButton.Cancel + ) msg.setDefaultButton(QMessageBox.StandardButton.Ok) - + result = msg.exec() return result == QMessageBox.StandardButton.Ok - + def show_error_dialog(self, error_message: str, title: str = "Error") -> None: """ Show error dialog with error icon. - + Args: error_message: Error message to display title: Dialog title @@ -229,11 +245,11 @@ def show_error_dialog(self, error_message: str, title: str = "Error") -> None: msg.setIcon(QMessageBox.Icon.Critical) msg.setStandardButtons(QMessageBox.StandardButton.Ok) msg.exec() - + def show_info_dialog(self, info_message: str, title: str = "Information") -> None: """ Show information dialog. - + Args: info_message: Information message to display title: Dialog title @@ -251,7 +267,7 @@ def show_cached_file_dialog( title: str = "Select File", file_filter: str = "All Files (*)", mode: str = "open", - fallback_path: Optional[Path] = None + fallback_path: Optional[Path] = None, ) -> Optional[Path]: """ Show file dialog with path caching (mirrors Textual TUI pattern). @@ -272,17 +288,11 @@ def show_cached_file_dialog( try: if mode == "save": file_path, _ = QFileDialog.getSaveFileName( - self.main_window, - title, - initial_dir, - file_filter + self.main_window, title, initial_dir, file_filter ) else: # mode == "open" file_path, _ = QFileDialog.getOpenFileName( - self.main_window, - title, - initial_dir, - file_filter + self.main_window, title, initial_dir, file_filter ) if file_path: @@ -302,7 +312,7 @@ def show_cached_directory_dialog( cache_key: PathCacheKey, title: str = "Select Directory", fallback_path: Optional[Path] = None, - allow_multiple: bool = False + allow_multiple: bool = False, ) -> Optional[Path | list[Path]]: """ Show directory dialog with path caching. @@ -319,25 +329,62 @@ def show_cached_directory_dialog( - List[Path] if allow_multiple=True """ # Get cached initial directory - initial_dir = str(get_cached_dialog_path(cache_key, fallback_path)) + initial_path = get_cached_dialog_path(cache_key, fallback_path) + initial_dir = str(initial_path) try: if allow_multiple: - # Use custom QFileDialog for multi-directory selection + # Use custom QFileDialog for multi-directory selection. + # NOTE: DontUseNativeDialog is required for multi-select. dialog = QFileDialog(self.main_window, title, initial_dir) dialog.setFileMode(QFileDialog.FileMode.Directory) dialog.setOption(QFileDialog.Option.DontUseNativeDialog, True) - # Enable multi-selection in the list view + # Enable multi-selection in the list/tree views. list_view = dialog.findChild(QWidget, "listView") if list_view: from PyQt6.QtWidgets import QAbstractItemView - list_view.setSelectionMode(QAbstractItemView.SelectionMode.ExtendedSelection) + + list_view.setSelectionMode( + QAbstractItemView.SelectionMode.ExtendedSelection + ) tree_view = dialog.findChild(QWidget, "treeView") if tree_view: from PyQt6.QtWidgets import QAbstractItemView - tree_view.setSelectionMode(QAbstractItemView.SelectionMode.ExtendedSelection) + + tree_view.setSelectionMode( + QAbstractItemView.SelectionMode.ExtendedSelection + ) + + # Make the path bar editable so users can paste paths. + # In the non-native dialog, the path widget is a QComboBox ("lookInCombo"). + try: + from PyQt6.QtWidgets import QComboBox + + look_in = dialog.findChild(QComboBox, "lookInCombo") + if look_in: + look_in.setEditable(True) + line_edit = look_in.lineEdit() + if line_edit: + line_edit.setPlaceholderText("Paste a path and press Enter") + + def _jump_to_typed_path() -> None: + raw = line_edit.text().strip().strip('"').strip("'") + if not raw: + return + try: + p = Path(raw).expanduser() + if p.exists() and p.is_dir(): + dialog.setDirectory(str(p)) + except Exception: + # Leave the dialog as-is if the path is invalid. + return + + line_edit.returnPressed.connect(_jump_to_typed_path) + except Exception: + # If the internal widget names differ across platforms, fail gracefully. + pass if dialog.exec(): selected_paths = [Path(p) for p in dialog.selectedFiles()] @@ -349,9 +396,7 @@ def show_cached_directory_dialog( else: # Single directory selection (native dialog) dir_path = QFileDialog.getExistingDirectory( - self.main_window, - title, - initial_dir + self.main_window, title, initial_dir ) if dir_path: @@ -369,17 +414,17 @@ def show_cached_directory_dialog( def run_system_command(self, command: str, wait_for_finish: bool = True) -> bool: """ Replace prompt_toolkit system command with QProcess. - + Args: command: System command to execute wait_for_finish: Whether to wait for command completion - + Returns: True if command executed successfully, False otherwise """ try: process = QProcess(self.main_window) - + if wait_for_finish: process.start(command) success = process.waitForFinished(30000) # 30 second timeout @@ -387,19 +432,19 @@ def run_system_command(self, command: str, wait_for_finish: bool = True) -> bool else: # Start detached process return process.startDetached(command) - + except Exception as e: logger.error(f"System command failed: {command} - {e}") self.show_error_dialog(f"Command failed: {e}") return False - + def open_external_editor(self, file_path: Path) -> bool: """ Open file in external editor using system default. - + Args: file_path: Path to file to edit - + Returns: True if editor opened successfully, False otherwise """ @@ -410,34 +455,34 @@ def open_external_editor(self, file_path: Path) -> bool: logger.error(f"Failed to open external editor: {e}") self.show_error_dialog(f"Failed to open editor: {e}") return False - + def get_global_config(self): """ Get global configuration from application. - + Returns: Global configuration object """ # Access global config through application property - if hasattr(self.app, 'global_config'): + if hasattr(self.app, "global_config"): return self.app.global_config else: # Fallback to default config - + return GlobalPipelineConfig() - + def set_global_config(self, config): """ Set global configuration on application. - + Args: config: Global configuration object """ - if hasattr(self.app, 'global_config'): + if hasattr(self.app, "global_config"): self.app.global_config = config else: # Set as application property - setattr(self.app, 'global_config', config) + setattr(self.app, "global_config", config) # ========== THEME MANAGEMENT METHODS ========== @@ -526,7 +571,7 @@ def register_theme_change_callback(self, callback): callback: Function to call with new color scheme """ self.theme_manager.register_theme_change_callback(callback) - + def get_file_manager(self): """ Get FileManager instance from application. @@ -534,14 +579,15 @@ def get_file_manager(self): Returns: FileManager instance """ - if hasattr(self.app, 'file_manager'): + if hasattr(self.app, "file_manager"): return self.app.file_manager else: # Create default FileManager from polystore.filemanager import FileManager from polystore.base import storage_registry + file_manager = FileManager(storage_registry) - setattr(self.app, 'file_manager', file_manager) + setattr(self.app, "file_manager", file_manager) return file_manager def get_event_bus(self) -> GlobalEventBus: @@ -556,31 +602,31 @@ def get_event_bus(self) -> GlobalEventBus: class ExternalEditorProcess(QThread): """ Thread for handling external editor processes. - + Replaces prompt_toolkit's run_system_command for external editor integration. """ - + finished = pyqtSignal(bool, str) # success, error_message - + def __init__(self, command: str, file_path: Path): super().__init__() self.command = command self.file_path = file_path - + def run(self): """Execute external editor command in thread.""" try: process = QProcess() process.start(self.command) - + success = process.waitForFinished(300000) # 5 minute timeout - + if success and process.exitCode() == 0: self.finished.emit(True, "") else: error_msg = process.readAllStandardError().data().decode() self.finished.emit(False, f"Editor failed: {error_msg}") - + except Exception as e: self.finished.emit(False, f"Editor process failed: {e}") @@ -588,35 +634,33 @@ def run(self): class AsyncOperationThread(QThread): """ Generic thread for async operations. - + Converts async operations to Qt thread-based operations. """ - + result_ready = pyqtSignal(object) error_occurred = pyqtSignal(str) - + def __init__(self, async_func, *args, **kwargs): super().__init__() self.async_func = async_func self.args = args self.kwargs = kwargs - + def run(self): """Execute async function in thread with event loop.""" try: import asyncio - + # Create new event loop for this thread loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - + # Run async function - result = loop.run_until_complete( - self.async_func(*self.args, **self.kwargs) - ) - + result = loop.run_until_complete(self.async_func(*self.args, **self.kwargs)) + self.result_ready.emit(result) - + except Exception as e: logger.error(f"Async operation failed: {e}") self.error_occurred.emit(str(e)) diff --git a/openhcs/pyqt_gui/services/time_travel_navigation.py b/openhcs/pyqt_gui/services/time_travel_navigation.py new file mode 100644 index 000000000..598d3370b --- /dev/null +++ b/openhcs/pyqt_gui/services/time_travel_navigation.py @@ -0,0 +1,107 @@ +"""Typed helpers for OpenHCS time-travel navigation decisions.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable, Literal, Optional + +from pyqt_reactive.services.function_navigation import ( + FUNCTION_FIELD_ROOT, + build_function_token_field_path, + is_function_field_path, +) + +FUNCTION_STEP_SCOPE_PREFIX = "functionstep_" + + +@dataclass(frozen=True) +class FunctionScopeRef: + """Typed parse result for function ObjectState scopes.""" + + step_scope_id: str + function_token: str + + +@dataclass(frozen=True) +class TimeTravelNavigationTarget: + """Typed time-travel navigation target.""" + + kind: Literal["field_path", "function_token"] + value: str + + @property + def is_function_target(self) -> bool: + return self.kind == "function_token" or is_function_field_path(self.value) + + def to_field_path(self) -> str: + if self.kind == "function_token": + return build_function_token_field_path(self.value) + return self.value + + +def parse_function_scope_ref(scope_id: str) -> FunctionScopeRef | None: + """Parse function ObjectState scope into typed reference.""" + parts = scope_id.rsplit("::", 2) + if len(parts) < 3: + return None + parent_scope, step_token, function_token = parts + if not step_token.startswith(FUNCTION_STEP_SCOPE_PREFIX): + return None + if not function_token: + return None + step_scope_id = f"{parent_scope}::{step_token}" + return FunctionScopeRef( + step_scope_id=step_scope_id, + function_token=function_token, + ) + + +def make_function_token_target(function_token: str) -> TimeTravelNavigationTarget: + """Create typed function-token navigation target.""" + return TimeTravelNavigationTarget( + kind="function_token", + value=function_token, + ) + + +def make_field_path_target(field_path: str) -> TimeTravelNavigationTarget: + """Create typed plain field-path navigation target.""" + return TimeTravelNavigationTarget( + kind="field_path", + value=field_path, + ) + + +def resolve_fallback_field_path( + last_changed_field: str | None, + dirty_fields: Iterable[str], +) -> Optional[str]: + """Resolve best-effort field path when no explicit function scope mapping exists.""" + if last_changed_field: + return last_changed_field + + candidates = [field for field in dirty_fields if isinstance(field, str)] + if not candidates: + return None + sorted_fields = sorted(candidates, key=_dirty_field_sort_key) + return sorted_fields[0] + + +def should_replace_navigation_target( + existing_target: TimeTravelNavigationTarget | None, + candidate_target: TimeTravelNavigationTarget | None, +) -> bool: + """Return True when candidate should replace existing target.""" + if existing_target is None: + return candidate_target is not None + if candidate_target is None: + return False + if existing_target.is_function_target: + return False + return candidate_target.is_function_target + + +def _dirty_field_sort_key(field: str) -> tuple[int, int, str]: + """Sort key prioritizing root function field then deeper paths.""" + function_rank = 0 if field == FUNCTION_FIELD_ROOT else 1 + return (function_rank, -field.count("."), field) diff --git a/openhcs/pyqt_gui/services/window_config.py b/openhcs/pyqt_gui/services/window_config.py new file mode 100644 index 000000000..8f57e711c --- /dev/null +++ b/openhcs/pyqt_gui/services/window_config.py @@ -0,0 +1,42 @@ +""" +Window Configuration - Declarative specs for application windows. + +Centralized window definitions replacing hardcoded creation code. +""" + +from dataclasses import dataclass +from typing import Type +from PyQt6.QtWidgets import QDialog, QVBoxLayout + + +class ManagedWindow(QDialog): + """Base class for managed application windows.""" + + def __init__(self, main_window, service_adapter): + super().__init__(main_window) + self.main_window = main_window + self.service_adapter = service_adapter + self.setup_ui() + self.setup_connections() + + def setup_ui(self): + """Setup window UI. Subclasses implement this.""" + pass + + def setup_connections(self): + """Setup signal connections. Subclasses implement this.""" + pass + + +@dataclass(frozen=True) +class WindowSpec: + """ + Declarative specification for an application window. + + Centralizes window configuration (widget, title, size) in one place. + """ + + window_id: str + title: str + window_class: Type[ManagedWindow] + initialize_on_startup: bool = False diff --git a/openhcs/pyqt_gui/services/window_factory.py b/openhcs/pyqt_gui/services/window_factory.py deleted file mode 100644 index 7e1763371..000000000 --- a/openhcs/pyqt_gui/services/window_factory.py +++ /dev/null @@ -1,191 +0,0 @@ -"""OpenHCS-specific window factory for scope-aware window creation. - -Implements WindowFactoryABC to provide openhcs-specific window creation logic -for global config, plate config, and step/function editor windows. -""" - -import logging -from typing import Optional, Any - -from PyQt6.QtWidgets import QWidget, QApplication - -from pyqt_reactive.protocols import WindowFactoryABC -from objectstate import ObjectStateRegistry - -logger = logging.getLogger(__name__) - - -class OpenHCSWindowFactory(WindowFactoryABC): - """OpenHCS implementation of window factory. - - Handles scope_id formats: - - "" (empty string): GlobalPipelineConfig - - "__plates__": Root plate list state (no window) - - "/path/to/plate": PipelineConfig for that plate - - "/path/to/plate::step_N": DualEditorWindow → Step Settings tab - - "/path/to/plate::step_N::func_M": DualEditorWindow → Function Pattern tab - """ - - def create_window_for_scope(self, scope_id: str, object_state: Optional[Any] = None) -> Optional[QWidget]: - """Create and show a window for the given scope_id.""" - if scope_id == "": - return self._create_global_config_window() - elif scope_id == "__plates__": - # __plates__ is the root plate list state - no window to create for it - # Time-travel changes to the plate list are reflected in PlateManager automatically - logger.debug(f"[WINDOW_FACTORY] Skipping window creation for __plates__ scope") - return None - elif "::" not in scope_id: - return self._create_plate_config_window(scope_id) - else: - return self._create_step_editor_window(scope_id, object_state) - - def _create_global_config_window(self) -> Optional[QWidget]: - """Create GlobalPipelineConfig editor window.""" - from openhcs.pyqt_gui.windows.config_window import ConfigWindow - from openhcs.core.config import GlobalPipelineConfig - from openhcs.config_framework.global_config import get_current_global_config, set_global_config_for_editing - - current_config = get_current_global_config(GlobalPipelineConfig) or GlobalPipelineConfig() - - def handle_save(new_config): - set_global_config_for_editing(GlobalPipelineConfig, new_config) - logger.info("Global config saved via window") - - window = ConfigWindow( - config_class=GlobalPipelineConfig, - current_config=current_config, - on_save_callback=handle_save, - scope_id="" - ) - window.show() - window.raise_() - window.activateWindow() - return window - - def _create_plate_config_window(self, scope_id: str) -> Optional[QWidget]: - """Create PipelineConfig editor window for a plate.""" - from openhcs.pyqt_gui.windows.config_window import ConfigWindow - from openhcs.core.config import PipelineConfig - - plate_manager = self._find_plate_manager() - if not plate_manager: - logger.warning("Could not find PlateManager for plate config window") - return None - - orchestrator = ObjectStateRegistry.get_object(scope_id) - if not orchestrator: - logger.warning(f"No orchestrator found for scope: {scope_id}") - return None - - window = ConfigWindow( - config_class=PipelineConfig, - current_config=orchestrator.pipeline_config, - on_save_callback=None, # ObjectState handles save - scope_id=scope_id - ) - window.show() - window.raise_() - window.activateWindow() - return window - - def _create_step_editor_window(self, scope_id: str, object_state: Optional[Any] = None) -> Optional[QWidget]: - """Create DualEditorWindow for step or function scope. - - Args: - scope_id: Step or function scope - object_state: If provided, get step from object_instance (for time-travel) - """ - from openhcs.pyqt_gui.windows.dual_editor_window import DualEditorWindow - - parts = scope_id.split("::") - if len(parts) < 2: - logger.warning(f"Invalid step scope_id format: {scope_id}") - return None - - plate_path = parts[0] - step_token = parts[1] - is_function_scope = len(parts) >= 3 - step_scope_id = f"{parts[0]}::{parts[1]}" - - plate_manager = self._find_plate_manager() - if not plate_manager: - logger.warning("Could not find PlateManager for step editor") - return None - - orchestrator = ObjectStateRegistry.get_object(plate_path) - if not orchestrator: - logger.warning(f"No orchestrator found for plate: {plate_path}") - return None - - # Get step: from ObjectState if provided, else find by token - step = None - if object_state: - # Time-travel: use object_instance directly - if is_function_scope: - # Function scope - get parent step's ObjectState - step_state = ObjectStateRegistry.get_by_scope(step_scope_id) - step = step_state.object_instance if step_state else None - else: - step = object_state.object_instance - - if not step: - # Provenance navigation: find by token - step = self._find_step_by_token(plate_manager, plate_path, step_token) - - if not step: - logger.warning(f"Could not find step for scope: {scope_id}") - return None - - window = DualEditorWindow( - step_data=step, - is_new=False, - on_save_callback=None, # ObjectState handles save - orchestrator=orchestrator, - parent=None - ) - if is_function_scope and window.tab_widget: - window.tab_widget.setCurrentIndex(1) - - window.show() - window.raise_() - window.activateWindow() - return window - - def _find_plate_manager(self): - """Find PlateManagerWidget from main window.""" - from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget - - for widget in QApplication.topLevelWidgets(): - if hasattr(widget, 'floating_windows'): - plate_dialog = widget.floating_windows.get("plate_manager") - if plate_dialog: - return plate_dialog.findChild(PlateManagerWidget) - return None - - def _find_step_by_token(self, plate_manager, plate_path: str, step_token: str): - """Find a step in the pipeline by its scope token.""" - # Get steps from Pipeline ObjectState instead of deprecated plate_pipelines - pipeline_scope = f"{plate_path}::pipeline" - pipeline_state = ObjectStateRegistry.get_by_scope(pipeline_scope) - - if not pipeline_state: - logger.debug(f"No pipeline state for {plate_path}") - return None - - step_scope_ids = pipeline_state.parameters.get("step_scope_ids") or [] - - for scope_id in step_scope_ids: - step_state = ObjectStateRegistry.get_by_scope(scope_id) - if step_state: - step = step_state.object_instance - token = getattr(step, '_scope_token', None) - if token == step_token: - return step - token2 = getattr(step, '_pipeline_scope_token', None) - if token2 == step_token: - return step - - logger.debug(f"Step token '{step_token}' not found in {len(step_scope_ids)} steps") - return None - diff --git a/openhcs/pyqt_gui/services/window_handlers.py b/openhcs/pyqt_gui/services/window_handlers.py new file mode 100644 index 000000000..96180bbf0 --- /dev/null +++ b/openhcs/pyqt_gui/services/window_handlers.py @@ -0,0 +1,249 @@ +"""OpenHCS window factory registration. + +Registers all OpenHCS-specific scope patterns with the generic WindowFactory. +Called during application initialization. +""" + +import logging +from typing import Optional, TYPE_CHECKING + +from PyQt6.QtWidgets import QWidget +from pyqt_reactive.services import ScopeWindowRegistry + +if TYPE_CHECKING: + from openhcs.config_framework.object_state import ObjectState + +# Import FunctionStep for type checking in tab selection +from openhcs.core.steps.function_step import FunctionStep + +logger = logging.getLogger(__name__) + + +def _create_global_config_window(scope_id: str, object_state=None) -> Optional[QWidget]: + """Create GlobalPipelineConfig editor window.""" + from openhcs.pyqt_gui.windows.config_window import ConfigWindow + from openhcs.core.config import GlobalPipelineConfig + from openhcs.config_framework.global_config import ( + get_current_global_config, + set_global_config_for_editing, + ) + + current_config = ( + get_current_global_config(GlobalPipelineConfig) or GlobalPipelineConfig() + ) + + def handle_save(new_config): + set_global_config_for_editing(GlobalPipelineConfig, new_config) + logger.info("Global config saved via window") + + window = ConfigWindow( + config_class=GlobalPipelineConfig, + current_config=current_config, + on_save_callback=handle_save, + scope_id=scope_id, + ) + window.show() + window.raise_() + window.activateWindow() + return window + + +def _create_plates_root_window(scope_id: str, object_state=None) -> Optional[QWidget]: + """Root plate list state - no window to create.""" + logger.debug(f"[WINDOW_FACTORY] Skipping window creation for __plates__ scope") + return None + + +def _create_plate_config_window(scope_id: str, object_state=None) -> Optional[QWidget]: + """Create PipelineConfig editor window for a plate.""" + from openhcs.pyqt_gui.windows.config_window import ConfigWindow + from openhcs.core.config import PipelineConfig + from pyqt_reactive.services import ServiceRegistry + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + from objectstate import ObjectStateRegistry + + # Get plate manager from ServiceRegistry + plate_manager = ServiceRegistry.get(PlateManagerWidget) + if not plate_manager: + logger.warning("Could not find PlateManager for plate config window") + return None + + orchestrator = ObjectStateRegistry.get_object(scope_id) + if not orchestrator: + logger.warning(f"No orchestrator found for scope: {scope_id}") + return None + + window = ConfigWindow( + config_class=PipelineConfig, + current_config=orchestrator.pipeline_config, + on_save_callback=None, # ObjectState handles save + scope_id=scope_id, + ) + window.show() + window.raise_() + window.activateWindow() + return window + + +def _create_step_editor_window( + scope_id: str, object_state: Optional["ObjectState"] = None +) -> Optional[QWidget]: + """Create DualEditorWindow for step or function scope.""" + from openhcs.pyqt_gui.windows.dual_editor_window import DualEditorWindow + from pyqt_reactive.services import ServiceRegistry + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + from objectstate import ObjectStateRegistry + + parts = scope_id.split("::") + if len(parts) < 2: + logger.warning(f"Invalid step scope_id format: {scope_id}") + return None + + plate_path = parts[0] + step_token = parts[1] + is_function_scope = len(parts) >= 3 + step_scope_id = f"{parts[0]}::{parts[1]}" + + # Get plate manager from ServiceRegistry + plate_manager = ServiceRegistry.get(PlateManagerWidget) + if not plate_manager: + logger.warning("Could not find PlateManager for step editor") + return None + + orchestrator = ObjectStateRegistry.get_object(plate_path) + if not orchestrator: + logger.warning(f"No orchestrator found for plate: {plate_path}") + return None + + # Get step: from ObjectState if provided, else find by token + step = None + if object_state: + if is_function_scope: + step_state = ObjectStateRegistry.get_by_scope(step_scope_id) + step = step_state.object_instance if step_state else None + else: + step = object_state.object_instance + + if not step: + step = _find_step_by_token(plate_manager, plate_path, step_token) + + if not step: + logger.warning(f"Could not find step for scope: {scope_id}") + return None + + window = DualEditorWindow( + step_data=step, + is_new=False, + on_save_callback=None, # ObjectState handles save + orchestrator=orchestrator, + parent=None, + ) + if window.tab_widget: + # Determine which tab to show based on the type of object being edited + # Look up ObjectState by scope_id to get the actual object instance type + state_for_tab_selection = ObjectStateRegistry.get_by_scope(scope_id) + if state_for_tab_selection: + obj_instance = state_for_tab_selection.object_instance + is_function_step = isinstance(obj_instance, FunctionStep) + logger.debug( + f"[TAB_SELECT] scope_id={scope_id}, object_type={type(obj_instance).__name__}, " + f"is_function_step={is_function_step}" + ) + if is_function_step: + # Editing a FunctionStep - show Step Settings tab + logger.debug( + f"[TAB_SELECT] Setting Step Settings tab (index 0) - FunctionStep instance" + ) + window.tab_widget.setCurrentIndex(0) + else: + # Editing something else (e.g., function pattern) - show Function Pattern tab + logger.debug( + f"[TAB_SELECT] Setting Function Pattern tab (index 1) - not a FunctionStep" + ) + window.tab_widget.setCurrentIndex(1) + else: + # Fallback: use old logic if we can't look up the state + logger.debug( + f"[TAB_SELECT] Fallback: scope_id={scope_id}, is_function_scope={is_function_scope}, " + f"dirty_fields={getattr(object_state, 'dirty_fields', None)}" + ) + if is_function_scope: + logger.debug( + f"[TAB_SELECT] Setting Function Pattern tab (index 1) for function scope (fallback)" + ) + window.tab_widget.setCurrentIndex(1) + elif object_state and "func" in object_state.dirty_fields: + logger.debug( + f"[TAB_SELECT] Setting Function Pattern tab (index 1) - func in dirty_fields (fallback)" + ) + window.tab_widget.setCurrentIndex(1) + else: + logger.debug( + f"[TAB_SELECT] Setting Step Settings tab (index 0) - default (fallback)" + ) + window.tab_widget.setCurrentIndex(0) + + window.show() + window.raise_() + window.activateWindow() + return window + + +def _find_step_by_token(plate_manager, plate_path: str, step_token: str): + """Find a step in the pipeline by its scope token.""" + from objectstate import ObjectStateRegistry + + pipeline_scope = f"{plate_path}::pipeline" + pipeline_state = ObjectStateRegistry.get_by_scope(pipeline_scope) + + if not pipeline_state: + logger.debug(f"No pipeline state for {plate_path}") + return None + + step_scope_ids = pipeline_state.parameters.get("step_scope_ids") or [] + + for scope_id in step_scope_ids: + step_state = ObjectStateRegistry.get_by_scope(scope_id) + if step_state: + step = step_state.object_instance + token = getattr(step, "_scope_token", None) + if token == step_token: + return step + token2 = getattr(step, "_pipeline_scope_token", None) + if token2 == step_token: + return step + + logger.debug(f"Step token '{step_token}' not found in {len(step_scope_ids)} steps") + return None + + +def register_openhcs_window_handlers(): + """Register all OpenHCS window handlers with the generic factory. + + Call this during application startup. + """ + # Order matters - more specific patterns should come first + + # Step/function editors (::functionstep_N or ::functionstep_N::func_M) + # Note: uses "functionstep" prefix derived from FunctionStep class name + ScopeWindowRegistry.register_handler( + pattern=r"^.*::functionstep_\d+(::func_\d+)?$", + handler=_create_step_editor_window, + ) + + # Plate configs (/path/to/plate - no :: in scope_id) + ScopeWindowRegistry.register_handler( + pattern=r"^/[^:]*$", handler=_create_plate_config_window + ) + + # Plates root list + ScopeWindowRegistry.register_handler( + pattern=r"^__plates__$", handler=_create_plates_root_window + ) + + # Global config (empty string) + ScopeWindowRegistry.register_handler( + pattern=r"^$", handler=_create_global_config_window + ) + + logger.info("[WINDOW_FACTORY] Registered OpenHCS window handlers") diff --git a/openhcs/pyqt_gui/widgets/image_browser.py b/openhcs/pyqt_gui/widgets/image_browser.py index 1c78e7bd1..06c7c0628 100644 --- a/openhcs/pyqt_gui/widgets/image_browser.py +++ b/openhcs/pyqt_gui/widgets/image_browser.py @@ -6,17 +6,31 @@ """ import logging -from dataclasses import dataclass, field +import time +import re from pathlib import Path from typing import Optional, List, Dict, Set, Any from PyQt6.QtWidgets import ( - QWidget, QVBoxLayout, QHBoxLayout, QTableWidgetItem, - QPushButton, QLabel, QHeaderView, QAbstractItemView, QMessageBox, - QSplitter, QGroupBox, QTreeWidget, QTreeWidgetItem, QScrollArea, - QLineEdit, QTabWidget, QTextEdit + QWidget, + QVBoxLayout, + QHBoxLayout, + QTableWidgetItem, + QPushButton, + QLabel, + QHeaderView, + QAbstractItemView, + QMessageBox, + QSplitter, + QGroupBox, + QTreeWidget, + QTreeWidgetItem, + QScrollArea, + QLineEdit, + QTabWidget, + QTextEdit, ) -from PyQt6.QtCore import Qt, pyqtSignal, pyqtSlot +from PyQt6.QtCore import Qt, pyqtSignal, pyqtSlot, QTimer from openhcs.constants.constants import Backend from polystore.filemanager import FileManager @@ -25,19 +39,45 @@ from pyqt_reactive.theming import StyleSheetGenerator from pyqt_reactive.widgets.shared.column_filter_widget import MultiColumnFilterPanel from pyqt_reactive.widgets.shared.image_table_browser import ImageTableBrowser +from pyqt_reactive.widgets.shared import TabbedFormWidget, TabConfig, TabbedFormConfig from openhcs.config_framework.object_state import ObjectState, ObjectStateRegistry -from openhcs.core.config import LazyNapariStreamingConfig, LazyFijiStreamingConfig +from openhcs.core.config import StreamingConfig +from objectstate.lazy_factory import get_base_type_for_lazy +from pyqt_reactive.forms import ParameterFormManager, FormManagerConfig + logger = logging.getLogger(__name__) -@dataclass -class ImageBrowserConfig: - """Namespace container for ImageBrowser's streaming configs. +def _get_viewer_display_name(field_name: str) -> str: + """Get display name for a streaming config field. + + Converts snake_case field name (e.g., napari_streaming_config) to + display name (e.g., Napari). + """ + # Remove '_streaming_config' suffix and convert to title case + viewer_name = field_name.replace("_streaming_config", "") + return viewer_name.replace("_", " ").title() + + +def _create_image_browser_config(): + """Create ImageBrowser config container with LAZY streaming configs. - Gives ImageBrowser a single root ObjectState with nested configs via dotted paths. + Uses SimpleNamespace with dynamically injected LAZY streaming configs. + Lazy configs resolve values from parent_state (plate) through the + ObjectState hierarchy, enabling live context updates. """ - napari_config: LazyNapariStreamingConfig = field(default_factory=LazyNapariStreamingConfig) - fiji_config: LazyFijiStreamingConfig = field(default_factory=LazyFijiStreamingConfig) + from types import SimpleNamespace + + config = SimpleNamespace() + + # Auto-discover streaming configs from registry + # Registry keys are now snake_case field names (e.g., 'napari_streaming_config') + for field_name in StreamingConfig.__registry__.keys(): + lazy_class = StreamingConfig.__registry__[field_name] + instance = lazy_class() # Lazy config resolves from plate via parent_state + setattr(config, field_name, instance) + + return config class ImageBrowserWidget(QWidget): @@ -50,9 +90,13 @@ class ImageBrowserWidget(QWidget): # Signals image_selected = pyqtSignal(str) # Emitted when an image is selected - _status_update_signal = pyqtSignal(str) # Internal signal for thread-safe status updates + _status_update_signal = pyqtSignal( + str + ) # Internal signal for thread-safe status updates - def __init__(self, orchestrator=None, color_scheme: Optional[ColorScheme] = None, parent=None): + def __init__( + self, orchestrator=None, color_scheme: Optional[ColorScheme] = None, parent=None + ): super().__init__(parent) self.orchestrator = orchestrator @@ -61,28 +105,38 @@ def __init__(self, orchestrator=None, color_scheme: Optional[ColorScheme] = None # Use orchestrator's filemanager if available, otherwise create a new one with global registry # This ensures the image browser can access all backends registered in the orchestrator's registry # (e.g., virtual_workspace backend) - self.filemanager = orchestrator.filemanager if orchestrator else FileManager(storage_registry) + self.filemanager = ( + orchestrator.filemanager if orchestrator else FileManager(storage_registry) + ) # Scope ID for cross-window live context filtering (make distinct from PipelineConfig window) # Append a suffix so image browser uses a separate scope per plate - self.scope_id: Optional[str] = f"{orchestrator.plate_path}::image_browser" if orchestrator else None + self.scope_id: Optional[str] = ( + f"{orchestrator.plate_path}::image_browser" if orchestrator else None + ) - # Create root ObjectState from ImageBrowserConfig namespace container + # Create root ObjectState from dynamically generated config container # This gives us a single registered state with nested configs via dotted paths - self.config = ImageBrowserConfig() - parent_state = ObjectStateRegistry.get_by_scope(self.scope_id) if self.scope_id else None + self.config = _create_image_browser_config() + parent_state = ( + ObjectStateRegistry.get_by_scope(self.scope_id) if self.scope_id else None + ) self.state = ObjectState( object_instance=self.config, scope_id=self.scope_id, parent_state=parent_state, ) # Register in ObjectStateRegistry for cross-window inheritance + # Use _skip_snapshot=True since this is hidden machinery, not user-facing state if self.scope_id: - ObjectStateRegistry.register(self.state) + ObjectStateRegistry.register(self.state, _skip_snapshot=True) + + # TabbedFormWidget will be created lazily in _create_right_panel + # to avoid heavy initialization during widget construction. + self.tabbed_form = None - # PFM widgets (will be created in init_ui) - self.napari_config_form = None - self.fiji_config_form = None + # View buttons - dictionary keyed by viewer_type for dynamic handling + self.view_buttons: Dict[str, QPushButton] = {} # File data tracking (images + results) self.all_files = {} # filename -> metadata dict (merged images + results) @@ -92,6 +146,7 @@ def __init__(self, orchestrator=None, color_scheme: Optional[ColorScheme] = None self.filtered_files = {} # filename -> metadata dict (after search/filter) self.selected_wells = set() # Selected wells for filtering self.metadata_keys = [] # Column names from parser metadata (union of all keys) + self._metadata_display_cache = {} # (metadata_key, value_str) -> display value # Plate view widget (will be created in init_ui) self.plate_view_widget = None @@ -111,6 +166,7 @@ def __init__(self, orchestrator=None, color_scheme: Optional[ColorScheme] = None self._streaming_service = None if orchestrator: from openhcs.ui.shared.streaming_service import StreamingService + self._streaming_service = StreamingService( filemanager=self.filemanager, microscope_handler=orchestrator.microscope_handler, @@ -124,7 +180,7 @@ def __init__(self, orchestrator=None, color_scheme: Optional[ColorScheme] = None # Load images if orchestrator is provided if self.orchestrator: - self.load_images() + QTimer.singleShot(0, self.load_images) def init_ui(self): """Initialize the user interface.""" @@ -169,7 +225,9 @@ def init_ui(self): # Info label (moved from bottom) self.info_label = QLabel("No images loaded") - self.info_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_disabled)};") + self.info_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_disabled)};" + ) search_layout.addWidget(self.info_label, 0) # No stretch layout.addLayout(search_layout) @@ -187,8 +245,12 @@ def init_ui(self): # Column filter panel (initially empty, populated when images load) # DO NOT wrap in scroll area - breaks splitter resizing! # Each filter has its own scroll area for checkboxes - self.column_filter_panel = MultiColumnFilterPanel(color_scheme=self.color_scheme) - self.column_filter_panel.filters_changed.connect(self._on_column_filters_changed) + self.column_filter_panel = MultiColumnFilterPanel( + color_scheme=self.color_scheme + ) + self.column_filter_panel.filters_changed.connect( + self._on_column_filters_changed + ) self.column_filter_panel.setVisible(False) # Hidden until images load left_splitter.addWidget(self.column_filter_panel) @@ -202,7 +264,10 @@ def init_ui(self): # Plate view (initially hidden) from openhcs.pyqt_gui.widgets.shared.plate_view_widget import PlateViewWidget - self.plate_view_widget = PlateViewWidget(color_scheme=self.color_scheme, parent=self) + + self.plate_view_widget = PlateViewWidget( + color_scheme=self.color_scheme, parent=self + ) self.plate_view_widget.wells_selected.connect(self._on_wells_selected) self.plate_view_widget.detach_requested.connect(self._detach_plate_view) self.plate_view_widget.setVisible(False) @@ -221,8 +286,9 @@ def init_ui(self): right_panel = self._create_right_panel() main_splitter.addWidget(right_panel) - # Set initial splitter sizes (20% tree, 50% middle, 30% config) - main_splitter.setSizes([200, 500, 300]) + # Set initial splitter sizes (100px left, flexible middle, 400px right) + # Middle uses large value so it takes remaining space proportionally + main_splitter.setSizes([100, 2000, 400]) # Add splitter with stretch factor to fill vertical space layout.addWidget(main_splitter, 1) @@ -250,12 +316,13 @@ def _create_table_widget(self): """Create and configure the unified file table widget (images + results).""" # Use ImageTableBrowser for unified table (multi-select, dynamic columns) self.image_table_browser = ImageTableBrowser( - color_scheme=self.color_scheme, - parent=self + color_scheme=self.color_scheme, parent=self ) # Connect signals - self.image_table_browser.item_double_clicked.connect(self._on_file_double_clicked) + self.image_table_browser.item_double_clicked.connect( + self._on_file_double_clicked + ) self.image_table_browser.items_selected.connect(self._on_files_selected) # Alias for backward compatibility during transition @@ -266,212 +333,107 @@ def _create_table_widget(self): # Removed _create_results_widget - now using unified file table def _create_right_panel(self): - """Create the right panel with config tabs and instance manager.""" + """Create the right panel with streaming config forms and instance manager. + + Uses TabbedFormWidget to show each streaming config in its own tab. + """ container = QWidget() - container.setMinimumWidth(300) # Prevent clipping of config widgets + container.setMinimumWidth( + 360 + ) # Wider minimum for better config visibility (80% increase from 200) layout = QVBoxLayout(container) layout.setContentsMargins(0, 0, 0, 0) layout.setSpacing(5) - # Tab bar row with view buttons - tab_row = QHBoxLayout() - tab_row.setContentsMargins(0, 0, 0, 0) - tab_row.setSpacing(5) - - # Tab widget for streaming configs - from PyQt6.QtWidgets import QTabWidget - self.streaming_tabs = QTabWidget() - self.streaming_tabs.setStyleSheet(self.style_gen.generate_tab_widget_style()) - - # Napari config panel (with enable checkbox) - napari_panel = self._create_napari_config_panel() - self.napari_tab_index = self.streaming_tabs.addTab(napari_panel, "Napari") - - # Fiji config panel (with enable checkbox) - fiji_panel = self._create_fiji_config_panel() - self.fiji_tab_index = self.streaming_tabs.addTab(fiji_panel, "Fiji") - - # Update tab text when configs are enabled/disabled - self._update_tab_labels() - - # Extract tab bar and add to horizontal layout - self.tab_bar = self.streaming_tabs.tabBar() - self.tab_bar.setExpanding(False) - self.tab_bar.setUsesScrollButtons(False) - tab_row.addWidget(self.tab_bar, 0) # No stretch - tabs at natural size - - # View buttons beside tabs - self.view_napari_btn = QPushButton("View in Napari") - self.view_napari_btn.clicked.connect(self.view_selected_in_napari) - self.view_napari_btn.setStyleSheet(self.style_gen.generate_button_style()) - self.view_napari_btn.setEnabled(False) - tab_row.addWidget(self.view_napari_btn, 0) # No stretch - - self.view_fiji_btn = QPushButton("View in Fiji") - self.view_fiji_btn.clicked.connect(self.view_selected_in_fiji) - self.view_fiji_btn.setStyleSheet(self.style_gen.generate_button_style()) - self.view_fiji_btn.setEnabled(False) - tab_row.addWidget(self.view_fiji_btn, 0) # No stretch - - layout.addLayout(tab_row) - - # Vertical splitter for configs and instance manager - vertical_splitter = QSplitter(Qt.Orientation.Vertical) - - # Extract the stacked widget (content area) from tab widget and add it to splitter - # The tab bar is already in tab_row above - from PyQt6.QtWidgets import QStackedWidget - stacked_widget = self.streaming_tabs.findChild(QStackedWidget) - if stacked_widget: - stacked_widget.setMinimumWidth(300) # Prevent clipping of config widgets - vertical_splitter.addWidget(stacked_widget) - - # Instance manager panel - instance_panel = self._create_instance_manager_panel() - vertical_splitter.addWidget(instance_panel) - - # Set initial sizes (80% configs, 20% instance manager) - vertical_splitter.setSizes([400, 100]) - - layout.addWidget(vertical_splitter) - - return container - - def _create_napari_config_panel(self): - """Create the Napari configuration panel with enable checkbox and lazy config widget.""" - from PyQt6.QtWidgets import QCheckBox - - panel = QGroupBox() - layout = QVBoxLayout(panel) - layout.setContentsMargins(5, 5, 5, 5) - layout.setSpacing(5) - - # Enable checkbox in header - self.napari_enable_checkbox = QCheckBox("Enable Napari Streaming") - self.napari_enable_checkbox.setChecked(True) # Enabled by default - self.napari_enable_checkbox.toggled.connect(self._on_napari_enable_toggled) - layout.addWidget(self.napari_enable_checkbox) - - # Create PFM scoped to napari_config using field_prefix - from pyqt_reactive.forms import ParameterFormManager, FormManagerConfig - - config = FormManagerConfig( - parent=panel, - color_scheme=self.color_scheme, - field_prefix='napari_config' # Scope to napari_config nested dataclass - ) - self.napari_config_form = ParameterFormManager( - state=self.state, # Share root ObjectState - config=config - ) - - # Wrap in scroll area for long forms (vertical scrolling only) - scroll = QScrollArea() - scroll.setWidgetResizable(True) - scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff) - scroll.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) - scroll.setWidget(self.napari_config_form) - layout.addWidget(scroll) - - return panel - - def _create_fiji_config_panel(self): - """Create the Fiji configuration panel with enable checkbox and lazy config widget.""" - from PyQt6.QtWidgets import QCheckBox - - panel = QGroupBox() - layout = QVBoxLayout(panel) - layout.setContentsMargins(5, 5, 5, 5) - layout.setSpacing(5) - - # Enable checkbox in header - self.fiji_enable_checkbox = QCheckBox("Enable Fiji Streaming") - self.fiji_enable_checkbox.setChecked(False) # Disabled by default - self.fiji_enable_checkbox.toggled.connect(self._on_fiji_enable_toggled) - layout.addWidget(self.fiji_enable_checkbox) - - # Create PFM scoped to fiji_config using field_prefix - from pyqt_reactive.forms import ParameterFormManager, FormManagerConfig - - config = FormManagerConfig( - parent=panel, + # Vertical splitter: tabbed config form on top, instance manager below + splitter = QSplitter(Qt.Orientation.Vertical) + splitter.setChildrenCollapsible(True) # Allow collapsing to 0 + + # Top panel: Tabbed streaming config forms + # Create view buttons for each streaming config (will be added to tab bar row) + header_widgets = [] + for field_name in StreamingConfig.__registry__.keys(): + display_name = _get_viewer_display_name(field_name) + btn = QPushButton(f"View in {display_name}") + btn.clicked.connect( + lambda checked, fn=field_name: self._view_selected_in_viewer(fn) + ) + btn.setStyleSheet(self.style_gen.generate_button_style()) + btn.setEnabled(False) + self.view_buttons[field_name] = btn + header_widgets.append(btn) + + # Create a tab for each streaming config type + tabs = [] + for field_name in StreamingConfig.__registry__.keys(): + display_name = _get_viewer_display_name(field_name) + tabs.append(TabConfig(name=display_name, field_ids=[field_name])) + + tabbed_config = TabbedFormConfig( + tabs=tabs, color_scheme=self.color_scheme, - field_prefix='fiji_config' # Scope to fiji_config nested dataclass - ) - self.fiji_config_form = ParameterFormManager( - state=self.state, # Share root ObjectState - config=config + use_scroll_area=True, # Each tab gets its own scroll area + header_widgets=header_widgets, # View buttons on same row as tabs ) - # Wrap in scroll area for long forms (vertical scrolling only) - scroll = QScrollArea() - scroll.setWidgetResizable(True) - scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff) - scroll.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) - scroll.setWidget(self.fiji_config_form) - layout.addWidget(scroll) + self.tabbed_form = TabbedFormWidget(state=self.state, config=tabbed_config) + splitter.addWidget(self.tabbed_form) - # Initially disable the form (checkbox is unchecked by default) - self.fiji_config_form.setEnabled(False) + # Connect to parameter changes to update view button states + self.tabbed_form.parameter_changed.connect(self._on_parameter_changed) - return panel + # Bottom panel: Instance manager (ZMQ server browser) + instance_panel = self._create_instance_manager_panel() + splitter.addWidget(instance_panel) - def _update_tab_labels(self): - """Update tab labels to show enabled/disabled status.""" - napari_enabled = self.napari_enable_checkbox.isChecked() - fiji_enabled = self.fiji_enable_checkbox.isChecked() + # Set initial splitter sizes (70% config, 30% instance manager) + splitter.setSizes([350, 150]) - napari_label = "Napari ✓" if napari_enabled else "Napari" - fiji_label = "Fiji ✓" if fiji_enabled else "Fiji" + layout.addWidget(splitter, 1) # stretch factor = 1 - self.streaming_tabs.setTabText(self.napari_tab_index, napari_label) - self.streaming_tabs.setTabText(self.fiji_tab_index, fiji_label) + return container - def _get_config_values(self, prefix: str) -> Dict[str, Any]: - """Get current values for a nested config, scoped by prefix. + def _is_viewer_enabled(self, viewer_type: str) -> bool: + """Check if a viewer is enabled by querying its 'enabled' field from ObjectState. Args: - prefix: The dotted path prefix (e.g., 'napari_config' or 'fiji_config') + viewer_type: The streaming config field name (e.g., 'napari_streaming_config') Returns: - Dict with prefix stripped from keys, e.g., {'port': 5555, 'host': 'localhost'} + True if the viewer's streaming config has enabled=True, False otherwise. """ - prefix_dot = f'{prefix}.' - result = {} - for path, value in self.state.parameters.items(): - if path.startswith(prefix_dot): - remainder = path[len(prefix_dot):] - # Only direct children (no nested dots in remainder) - if '.' not in remainder: - result[remainder] = value - return result - - def _on_napari_enable_toggled(self, checked: bool): - """Handle Napari enable checkbox toggle.""" - self.napari_config_form.setEnabled(checked) - has_selection = len(self.image_table_browser.get_selected_keys()) > 0 - self.view_napari_btn.setEnabled(checked and has_selection) - self._update_tab_labels() + # viewer_type is already the field name (e.g., 'napari_streaming_config') + enabled_path = f"{viewer_type}.enabled" + # Get the resolved value (respects inheritance from parent_state) + return self.state.get_resolved_value(enabled_path) is True - def _on_fiji_enable_toggled(self, checked: bool): - """Handle Fiji enable checkbox toggle.""" - self.fiji_config_form.setEnabled(checked) - has_selection = len(self.image_table_browser.get_selected_keys()) > 0 - self.view_fiji_btn.setEnabled(checked and has_selection) - self._update_tab_labels() + def _get_enabled_viewers(self) -> list: + """Get list of all enabled viewer types. + + Returns: + List of viewer type strings (e.g., ['napari', 'fiji']) where enabled=True. + """ + return [ + viewer_type + for viewer_type in StreamingConfig.__registry__.keys() + if self._is_viewer_enabled(viewer_type) + ] def _create_instance_manager_panel(self): """Create the viewer instance manager panel using ZMQServerManagerWidget.""" - from openhcs.pyqt_gui.widgets.shared.zmq_server_manager import ZMQServerManagerWidget + from openhcs.pyqt_gui.widgets.shared.zmq_server_manager import ( + ZMQServerManagerWidget, + ) from openhcs.core.config import get_all_streaming_ports # Scan all streaming ports using orchestrator's pipeline config # This ensures we find viewers launched with custom ports # Exclude execution server port (only want viewer ports) from openhcs.constants.constants import DEFAULT_EXECUTION_SERVER_PORT + all_ports = get_all_streaming_ports( config=self.orchestrator.pipeline_config if self.orchestrator else None, - num_ports_per_type=10 + num_ports_per_type=10, ) ports_to_scan = [p for p in all_ports if p != DEFAULT_EXECUTION_SERVER_PORT] @@ -480,7 +442,7 @@ def _create_instance_manager_panel(self): ports_to_scan=ports_to_scan, title="Viewer Instances", style_generator=self.style_gen, - parent=self + parent=self, ) return zmq_manager @@ -489,7 +451,9 @@ def set_orchestrator(self, orchestrator): """Set the orchestrator and load images.""" self.orchestrator = orchestrator # CRITICAL: Preserve ::image_browser suffix to avoid scope conflicts with ConfigWindow - self.scope_id = f"{orchestrator.plate_path}::image_browser" if orchestrator else None + self.scope_id = ( + f"{orchestrator.plate_path}::image_browser" if orchestrator else None + ) # Use orchestrator's FileManager (has plate-specific backends like VirtualWorkspaceBackend) if orchestrator: @@ -500,10 +464,10 @@ def set_orchestrator(self, orchestrator): if self.state and orchestrator: self.state.context_obj = orchestrator.pipeline_config self.state.scope_id = self.scope_id - if self.napari_config_form: - self.napari_config_form._refresh_all_placeholders() - if self.fiji_config_form: - self.fiji_config_form._refresh_all_placeholders() + # Refresh form placeholders for all PFMs in tabs + if self.tabbed_form: + for form in self.tabbed_form.get_all_forms(): + form._refresh_all_placeholders() self.load_images() @@ -528,67 +492,89 @@ def on_folder_selection_changed(self): self._update_plate_view() def _apply_combined_filters(self): - """Apply search, folder, well, and column filters together.""" - # Start with search-filtered files - result = self.filtered_files.copy() - - # Apply folder filter if a folder is selected + """Apply search, folder, well, and column filters together in single pass.""" + # Get folder filter if selected selected_items = self.folder_tree.selectedItems() + folder_path = None + results_folder_path = None if selected_items: folder_path = selected_items[0].data(0, Qt.ItemDataRole.UserRole) - if folder_path: # Not root - # Filter by folder - include both the selected folder AND its associated results folder - # e.g., if "images" is selected, also include "images_results" + if folder_path: results_folder_path = f"{folder_path}_results" - result = { - filename: metadata for filename, metadata in result.items() - if (str(Path(filename).parent) == folder_path or - filename.startswith(folder_path + "/") or - str(Path(filename).parent) == results_folder_path or - filename.startswith(results_folder_path + "/")) + # Get active column filters (except Well if plate view is filtering) + active_filters = None + if self.column_filter_panel: + active_filters = self.column_filter_panel.get_active_filters() + if active_filters and self.selected_wells and "Well" in active_filters: + active_filters = { + k: v for k, v in active_filters.items() if k != "Well" } + logger.debug( + "[FILTER] Skipping Well column filter (plate view is filtering)" + ) - # Apply well filter if wells are selected - if self.selected_wells: - result = { - filename: metadata for filename, metadata in result.items() - if self._matches_wells(filename, metadata) - } + # Single-pass filtering: check all conditions for each file + result = {} + for filename, metadata in self.filtered_files.items(): + include = True + + # Folder filter + if folder_path and include: + include = ( + str(Path(filename).parent) == folder_path + or filename.startswith(folder_path + "/") + or str(Path(filename).parent) == results_folder_path + or filename.startswith(results_folder_path + "/") + ) - # Apply column filters - if self.column_filter_panel: - active_filters = self.column_filter_panel.get_active_filters() - if active_filters: - # Filter with AND logic across columns - filtered_result = {} - for filename, metadata in result.items(): - matches = True - for column_name, selected_values in active_filters.items(): - # Get the metadata key (lowercase with underscores) - metadata_key = column_name.lower().replace(' ', '_') - raw_value = metadata.get(metadata_key, '') - # Get display value for comparison (metadata name if available) - item_value = self._get_metadata_display_value(metadata_key, raw_value) - if item_value not in selected_values: - matches = False - break - if matches: - filtered_result[filename] = metadata - result = filtered_result - - # Update table with combined filters - self._populate_table(result) + # Well filter + if self.selected_wells and include: + include = self._matches_wells(filename, metadata) + + # Column filters (using pre-computed display values for speed) + if active_filters and include: + for column_name, selected_values in active_filters.items(): + metadata_key = column_name.lower().replace(" ", "_") + # Use pre-computed display value directly from metadata + item_value = metadata.get(f"_display_{metadata_key}", "") + if item_value not in selected_values: + include = False + break + + if include: + result[filename] = metadata + + # Update table with filtered results + self._update_table_with_filtered_items(result) logger.debug(f"Combined filters: {len(result)} images shown") + def _precompute_display_values(self): + """Pre-compute display values for all metadata keys in all files. + + This pre-computes values like "1 | W1" (raw | display_name) during load + to avoid repeated lookups during filtering. Store as "_display_{key}" in metadata. + """ + for metadata in self.all_files.values(): + for metadata_key in self.metadata_keys: + raw_value = metadata.get(metadata_key) + if raw_value is not None: + display_value = self._get_metadata_display_value( + metadata_key, raw_value + ) + metadata[f"_display_{metadata_key}"] = display_value + def _get_metadata_display_value(self, metadata_key: str, raw_value: Any) -> str: """ - Get display value for metadata, using metadata cache if available. + Get display value for metadata, using pre-computed values from metadata dict. For components like channel, this returns "1 | W1" format (raw key | metadata name) to preserve both the number and the metadata name. This handles cases where different subdirectories might have the same channel number mapped to different names. + First checks for pre-computed "_display_{key}" in metadata (fast path during filtering), + otherwise computes and caches the value. + Args: metadata_key: Metadata key (e.g., "channel", "site", "well") raw_value: Raw value from parser (e.g., 1, 2, "A01") @@ -598,37 +584,65 @@ def _get_metadata_display_value(self, metadata_key: str, raw_value: Any) -> str: otherwise just "raw_key" """ if raw_value is None: - return 'N/A' + return "N/A" # Convert to string for lookup value_str = str(raw_value) + # First check cache from pre-computed values (fast path during filtering) + cache_key = (metadata_key, value_str) + cached_value = self._metadata_display_cache.get(cache_key) + if cached_value is not None: + return cached_value + + # Compute and cache value + display_value = self._get_metadata_display_value_impl( + metadata_key, value_str, cache_key + ) + return display_value + + def _get_metadata_display_value_impl( + self, metadata_key: str, value_str: str, cache_key: tuple + ) -> str: + """Implementation of metadata display value computation.""" # Try to get metadata display name from cache if self.orchestrator: try: # Map metadata_key to AllComponents enum from openhcs.constants import AllComponents + component_map = { - 'channel': AllComponents.CHANNEL, - 'site': AllComponents.SITE, - 'z_index': AllComponents.Z_INDEX, - 'timepoint': AllComponents.TIMEPOINT, - 'well': AllComponents.WELL, + "channel": AllComponents.CHANNEL, + "site": AllComponents.SITE, + "z_index": AllComponents.Z_INDEX, + "timepoint": AllComponents.TIMEPOINT, + "well": AllComponents.WELL, } component = component_map.get(metadata_key) if component: - metadata_name = self.orchestrator._metadata_cache_service.get_component_metadata(component, value_str) - if metadata_name: + metadata_name = self.orchestrator._metadata_cache_service.get_component_metadata( + component, value_str + ) + if metadata_name and metadata_name != "None": # Format like TUI: "Channel 1 | HOECHST 33342" # But for table cells, just show "1 | W1" (more compact) - return f"{value_str} | {metadata_name}" - else: - logger.debug(f"No metadata name found for {metadata_key} {value_str}") + display_value = f"{value_str} | {metadata_name}" + self._metadata_display_cache[cache_key] = display_value + return display_value + logger.debug( + f"No metadata name found for {metadata_key} {value_str}" + ) except Exception as e: - logger.warning(f"Could not get metadata for {metadata_key} {value_str}: {e}", exc_info=True) + logger.warning( + f"Could not get metadata for {metadata_key} {value_str}: {e}", + exc_info=True, + ) + self._metadata_display_cache[cache_key] = value_str + return value_str # Fallback to raw value only + self._metadata_display_cache[cache_key] = value_str return value_str def _build_column_filters(self): @@ -646,27 +660,33 @@ def _build_column_filters(self): value = metadata.get(metadata_key) if value is not None: # Use metadata display value instead of raw value - display_value = self._get_metadata_display_value(metadata_key, value) + display_value = self._get_metadata_display_value( + metadata_key, value + ) unique_values.add(display_value) if unique_values: # Create filter for this column - column_display_name = metadata_key.replace('_', ' ').title() - self.column_filter_panel.add_column_filter(column_display_name, sorted(list(unique_values))) + column_display_name = metadata_key.replace("_", " ").title() + self.column_filter_panel.add_column_filter( + column_display_name, sorted(list(unique_values)) + ) # Show filter panel if we have filters if self.column_filter_panel.column_filters: self.column_filter_panel.setVisible(True) # Connect well filter to plate view for bidirectional sync - if 'Well' in self.column_filter_panel.column_filters and self.plate_view_widget: - well_filter = self.column_filter_panel.column_filters['Well'] + if "Well" in self.column_filter_panel.column_filters and self.plate_view_widget: + well_filter = self.column_filter_panel.column_filters["Well"] self.plate_view_widget.set_well_filter_widget(well_filter) # Connect well filter changes to sync back to plate view well_filter.filter_changed.connect(self._on_well_filter_changed) - logger.debug(f"Built {len(self.column_filter_panel.column_filters)} column filters") + logger.debug( + f"Built {len(self.column_filter_panel.column_filters)} column filters" + ) def _on_column_filters_changed(self): """Handle column filter changes.""" @@ -687,10 +707,10 @@ def filter_images(self, search_term: str): def create_searchable_text(metadata): """Create searchable text from file metadata.""" # 'filename' is guaranteed to exist (set in load_images/load_results) - searchable_fields = [metadata['filename']] + searchable_fields = [metadata["filename"]] # Add all metadata values for key, value in metadata.items(): - if key != 'filename' and value is not None: + if key != "filename" and value is not None: searchable_fields.append(str(value)) return " ".join(str(field) for field in searchable_fields) @@ -698,7 +718,7 @@ def create_searchable_text(metadata): if self._search_service is None: self._search_service = SearchService( all_items=self.all_files, - searchable_text_extractor=create_searchable_text + searchable_text_extractor=create_searchable_text, ) else: # Update search service with current files @@ -719,17 +739,24 @@ def load_images(self): return try: + self._metadata_display_cache.clear() logger.info("IMAGE BROWSER: Starting load_images()") # Get metadata handler from orchestrator handler = self.orchestrator.microscope_handler metadata_handler = handler.metadata_handler - logger.info(f"IMAGE BROWSER: Got metadata handler: {type(metadata_handler).__name__}") + logger.info( + f"IMAGE BROWSER: Got metadata handler: {type(metadata_handler).__name__}" + ) # Get image files from metadata (all subdirectories for browsing) plate_path = self.orchestrator.plate_path - logger.info(f"IMAGE BROWSER: Calling get_image_files for plate: {plate_path}") + logger.info( + f"IMAGE BROWSER: Calling get_image_files for plate: {plate_path}" + ) image_files = metadata_handler.get_image_files(plate_path, all_subdirs=True) - logger.info(f"IMAGE BROWSER: get_image_files returned {len(image_files) if image_files else 0} files") + logger.info( + f"IMAGE BROWSER: get_image_files returned {len(image_files) if image_files else 0} files" + ) if not image_files: self.info_label.setText("No images found") @@ -755,16 +782,14 @@ def load_images(self): else: size_str = "N/A" - metadata = { - 'filename': filename, - 'type': 'Image', - 'size': size_str - } + metadata = {"filename": filename, "type": "Image", "size": size_str} if parsed: metadata.update(parsed) self.all_images[filename] = metadata - logger.info(f"IMAGE BROWSER: Built all_images dict with {len(self.all_images)} entries") + logger.info( + f"IMAGE BROWSER: Built all_images dict with {len(self.all_images)} entries" + ) except Exception as e: logger.error(f"Failed to load images: {e}", exc_info=True) @@ -784,31 +809,54 @@ def load_images(self): all_keys.update(file_metadata.keys()) # Remove 'filename' from keys (it's always the first column) - all_keys.discard('filename') + all_keys.discard("filename") # Sort keys for consistent column order (extension first, then alphabetical) - self.metadata_keys = sorted(all_keys, key=lambda k: (k != 'extension', k)) + self.metadata_keys = sorted(all_keys, key=lambda k: (k != "extension", k)) # Configure ImageTableBrowser with dynamic columns self.image_table_browser.set_metadata_keys(self.metadata_keys) + # Pre-compute display values for fast filtering + self._precompute_display_values() + # Initialize filtered files to all files self.filtered_files = self.all_files.copy() # Build folder tree from file paths + folder_start = time.perf_counter() self._build_folder_tree() + logger.info( + "IMAGE BROWSER: Built folder tree in %.3fs", + time.perf_counter() - folder_start, + ) - # Build column filters from metadata - self._build_column_filters() - - # Populate table + # Populate table first to keep UI responsive + populate_start = time.perf_counter() self._populate_table(self.filtered_files) + logger.info( + "IMAGE BROWSER: Populated table in %.3fs", + time.perf_counter() - populate_start, + ) + + # Build column filters after initial render + def _build_filters_later(): + filters_start = time.perf_counter() + self._build_column_filters() + logger.info( + "IMAGE BROWSER: Built column filters in %.3fs", + time.perf_counter() - filters_start, + ) + + QTimer.singleShot(0, _build_filters_later) # Update info label total_files = len(self.all_files) num_images = len(self.all_images) num_results = len(self.all_results) - self.info_label.setText(f"{total_files} files loaded ({num_images} images, {num_results} results)") + self.info_label.setText( + f"{total_files} files loaded ({num_images} images, {num_results} results)" + ) # Update plate view if visible if self.plate_view_widget and self.plate_view_widget.isVisible(): @@ -834,7 +882,9 @@ def load_results(self): metadata_path = get_metadata_path(plate_path) if not metadata_path.exists(): - logger.warning(f"IMAGE BROWSER RESULTS: Metadata file not found: {metadata_path}") + logger.warning( + f"IMAGE BROWSER RESULTS: Metadata file not found: {metadata_path}" + ) self.all_results = {} return @@ -843,18 +893,27 @@ def load_results(self): # Collect ALL results directories from ALL subdirectories results_dirs = [] - if metadata and 'subdirectories' in metadata: - for subdir_name, subdir_metadata in metadata['subdirectories'].items(): - if 'results_dir' in subdir_metadata and subdir_metadata['results_dir']: - results_dir_path = plate_path / subdir_metadata['results_dir'] + if metadata and "subdirectories" in metadata: + for subdir_name, subdir_metadata in metadata["subdirectories"].items(): + if ( + "results_dir" in subdir_metadata + and subdir_metadata["results_dir"] + ): + results_dir_path = plate_path / subdir_metadata["results_dir"] results_dirs.append((subdir_name, results_dir_path)) - logger.info(f"IMAGE BROWSER RESULTS: Found results_dir for subdirectory '{subdir_name}': {subdir_metadata['results_dir']}") + logger.info( + f"IMAGE BROWSER RESULTS: Found results_dir for subdirectory '{subdir_name}': {subdir_metadata['results_dir']}" + ) if not results_dirs: - logger.warning("IMAGE BROWSER RESULTS: No results_dir found in any subdirectory") + logger.warning( + "IMAGE BROWSER RESULTS: No results_dir found in any subdirectory" + ) return - logger.info(f"IMAGE BROWSER RESULTS: Scanning {len(results_dirs)} results directories") + logger.info( + f"IMAGE BROWSER RESULTS: Scanning {len(results_dirs)} results directories" + ) # Get parser from orchestrator for filename parsing handler = self.orchestrator.microscope_handler @@ -863,33 +922,47 @@ def load_results(self): file_count = 0 for subdir_name, results_dir in results_dirs: if not results_dir.exists(): - logger.warning(f"IMAGE BROWSER RESULTS: Results directory does not exist: {results_dir}") + logger.warning( + f"IMAGE BROWSER RESULTS: Results directory does not exist: {results_dir}" + ) continue - logger.info(f"IMAGE BROWSER RESULTS: Scanning results directory for '{subdir_name}': {results_dir}") + logger.info( + f"IMAGE BROWSER RESULTS: Scanning results directory for '{subdir_name}': {results_dir}" + ) # Scan for ROI JSON files and CSV files - for file_path in results_dir.rglob('*'): + for file_path in results_dir.rglob("*"): if file_path.is_file(): file_count += 1 suffix = file_path.suffix.lower() - logger.debug(f"IMAGE BROWSER RESULTS: Found file: {file_path.name} (suffix={suffix})") + logger.debug( + f"IMAGE BROWSER RESULTS: Found file: {file_path.name} (suffix={suffix})" + ) # Determine file type using FileFormat registry from openhcs.constants.constants import FileFormat file_type = None - if file_path.name.endswith('.roi.zip'): - file_type = 'ROI' - logger.info(f"IMAGE BROWSER RESULTS: ✓ Matched as ROI: {file_path.name}") + if file_path.name.endswith(".roi.zip"): + file_type = "ROI" + logger.info( + f"IMAGE BROWSER RESULTS: ✓ Matched as ROI: {file_path.name}" + ) elif suffix in FileFormat.CSV.value: - file_type = 'CSV' - logger.info(f"IMAGE BROWSER RESULTS: ✓ Matched as CSV: {file_path.name}") + file_type = "CSV" + logger.info( + f"IMAGE BROWSER RESULTS: ✓ Matched as CSV: {file_path.name}" + ) elif suffix in FileFormat.JSON.value: - file_type = 'JSON' - logger.info(f"IMAGE BROWSER RESULTS: ✓ Matched as JSON: {file_path.name}") + file_type = "JSON" + logger.info( + f"IMAGE BROWSER RESULTS: ✓ Matched as JSON: {file_path.name}" + ) else: - logger.debug(f"IMAGE BROWSER RESULTS: ✗ Filtered out: {file_path.name} (suffix={suffix})") + logger.debug( + f"IMAGE BROWSER RESULTS: ✗ Filtered out: {file_path.name} (suffix={suffix})" + ) if file_type: # Get relative path from plate_path (not results_dir) to include subdirectory @@ -909,27 +982,37 @@ def load_results(self): # Build file info with parsed metadata (no full_path in metadata dict) file_info = { - 'filename': str(rel_path), - 'type': file_type, - 'size': size_str, + "filename": str(rel_path), + "type": file_type, + "size": size_str, } # Add parsed metadata components if parsing succeeded if parsed: file_info.update(parsed) - logger.info(f"IMAGE BROWSER RESULTS: ✓ Parsed result: {file_path.name} -> {parsed}") - logger.info(f"IMAGE BROWSER RESULTS: Full file_info: {file_info}") + logger.info( + f"IMAGE BROWSER RESULTS: ✓ Parsed result: {file_path.name} -> {parsed}" + ) + logger.info( + f"IMAGE BROWSER RESULTS: Full file_info: {file_info}" + ) else: - logger.warning(f"IMAGE BROWSER RESULTS: ✗ Could not parse filename: {file_path.name}") + logger.warning( + f"IMAGE BROWSER RESULTS: ✗ Could not parse filename: {file_path.name}" + ) # Store file info and full path separately self.all_results[str(rel_path)] = file_info self.result_full_paths[str(rel_path)] = file_path - logger.info(f"IMAGE BROWSER RESULTS: Scanned {file_count} total files, matched {len(self.all_results)} result files") + logger.info( + f"IMAGE BROWSER RESULTS: Scanned {file_count} total files, matched {len(self.all_results)} result files" + ) except Exception as e: - logger.error(f"IMAGE BROWSER RESULTS: Failed to load results: {e}", exc_info=True) + logger.error( + f"IMAGE BROWSER RESULTS: Failed to load results: {e}", exc_info=True + ) # Removed _populate_results_table - now using unified _populate_table # Removed on_result_double_clicked - now using unified on_file_double_clicked @@ -943,77 +1026,66 @@ def _stream_roi_file(self, roi_zip_path: Path): - Spawns background workers that do all heavy ROI loading + streaming. """ try: - # Check which viewers are enabled (cheap, safe on UI thread) - napari_enabled = self.napari_enable_checkbox.isChecked() - fiji_enabled = self.fiji_enable_checkbox.isChecked() + # Check which viewers are enabled by querying ObjectState + enabled_viewers = self._get_enabled_viewers() - if not napari_enabled and not fiji_enabled: + if not enabled_viewers: QMessageBox.information( self, "No Viewer Enabled", - "Please enable Napari or Fiji streaming to view ROIs." + "Please enable at least one viewer streaming to view ROIs.", ) return if not self.orchestrator: raise RuntimeError("No orchestrator set") - from openhcs.config_framework.context_manager import config_context - from openhcs.config_framework.lazy_factory import ( - resolve_lazy_configurations_for_serialization, - ) from openhcs.constants.constants import Backend as BackendEnum + from openhcs.pyqt_gui.utils.threading_utils import spawn_thread_with_context # For each enabled viewer, resolve config + viewer on UI thread, then spawn worker - if napari_enabled: - current_values = self._get_config_values('napari_config') - # CRITICAL: Pass None values through for lazy resolution - temp_config = LazyNapariStreamingConfig(**current_values) - - with config_context(self.orchestrator.pipeline_config): - with config_context(temp_config): - napari_config = resolve_lazy_configurations_for_serialization(temp_config) - - napari_viewer = self.orchestrator.get_or_create_visualizer(napari_config) - - import threading - - threading.Thread( - target=self._stream_single_roi_async, - args=( - napari_viewer, - roi_zip_path, - napari_config, - BackendEnum.NAPARI_STREAM, - "napari", - ), - daemon=True, - ).start() - - if fiji_enabled: - current_values = self._get_config_values('fiji_config') - # CRITICAL: Pass None values through for lazy resolution - temp_config = LazyFijiStreamingConfig(**current_values) - - with config_context(self.orchestrator.pipeline_config): - with config_context(temp_config): - fiji_config = resolve_lazy_configurations_for_serialization(temp_config) - - fiji_viewer = self.orchestrator.get_or_create_visualizer(fiji_config) - - import threading - - threading.Thread( - target=self._stream_single_roi_async, - args=( - fiji_viewer, - roi_zip_path, - fiji_config, - BackendEnum.FIJI_STREAM, - "fiji", - ), - daemon=True, - ).start() + for viewer_type in enabled_viewers: + # Get fully resolved streaming config from ObjectState (includes inheritance) + config = self.state.get_resolved_value(viewer_type) + + # Get the appropriate backend enum + backend_enum = getattr( + BackendEnum, f"{viewer_type.upper()}_STREAM", None + ) + if not backend_enum: + logger.error(f"No backend enum for viewer type: {viewer_type}") + continue + + # Create closure to capture viewer_config + def _make_acquire_and_stream(cfg, vt): + def _acquire_and_stream(): + try: + viewer = self.orchestrator.get_or_create_visualizer(cfg) + # _stream_single_roi_async itself starts a worker thread, + # so we call it here to kick off the streaming flow. + self._stream_single_roi_async( + viewer, roi_zip_path, cfg, backend_enum, vt + ) + except Exception as e: + logger.error( + f"Failed to acquire {vt} viewer or start streaming: {e}" + ) + from PyQt6.QtCore import QTimer + + QTimer.singleShot( + 0, + lambda vt=vt, e=e: QMessageBox.warning( + self, "Error", f"Failed to stream ROI to {vt}: {e}" + ), + ) + + return _acquire_and_stream + + # Spawn the thread with captured config + spawn_thread_with_context( + _make_acquire_and_stream(config, viewer_type), + name=f"acquire_{viewer_type}", + ) logger.info(f"Started async streaming of ROI file {roi_zip_path.name}") @@ -1021,8 +1093,9 @@ def _stream_roi_file(self, roi_zip_path: Path): logger.error(f"Failed to start ROI streaming: {e}") QMessageBox.warning(self, "Error", f"Failed to stream ROI file: {e}") - - def _stream_single_roi_async(self, viewer, roi_zip_path: Path, config, backend_enum, viewer_type: str): + def _stream_single_roi_async( + self, viewer, roi_zip_path: Path, config, backend_enum, viewer_type: str + ): """Worker: load a single ROI file and stream to a viewer in a background thread. Heavy operations only: @@ -1030,7 +1103,7 @@ def _stream_single_roi_async(self, viewer, roi_zip_path: Path, config, backend_e - viewer.wait_for_ready (long timeout) - filemanager.save """ - import threading + from objectstate import spawn_thread_with_context def _worker(): try: @@ -1049,7 +1122,9 @@ def _worker(): self._status_update_signal.emit(msg) # Show info dialog on UI thread - QTimer.singleShot(0, lambda: QMessageBox.information(self, "No ROIs", msg)) + QTimer.singleShot( + 0, lambda: QMessageBox.information(self, "No ROIs", msg) + ) return # Wait for viewer to be ready (never on UI thread) @@ -1102,15 +1177,13 @@ def _worker(): # Route to appropriate error dialog on UI thread if viewer_type == "napari": - QTimer.singleShot( - 0, lambda: self._show_streaming_error(str(e)) - ) + QTimer.singleShot(0, lambda: self._show_streaming_error(str(e))) else: QTimer.singleShot( 0, lambda: self._show_fiji_streaming_error(str(e)) ) - threading.Thread(target=_worker, daemon=True).start() + spawn_thread_with_context(_worker, name="stream_roi") def _populate_table(self, files_dict: Dict[str, Dict]): """Populate table with files (images + results) using ImageTableBrowser.""" @@ -1121,6 +1194,19 @@ def _populate_table(self, files_dict: Dict[str, Dict]): filtered = len(files_dict) self.image_table_browser.status_label.setText(f"Files: {filtered}/{total}") + def _update_table_with_filtered_items(self, files_dict: Dict[str, Dict]): + """Update table with filtered items without rebuilding SearchService. + + Use this when all_files has not changed, only filter criteria changed. + Much faster than _populate_table() for checkbox filter updates. + """ + self.image_table_browser.set_filtered_items(files_dict) + + # Update status label with file counts + total = len(self.all_files) + filtered = len(files_dict) + self.image_table_browser.status_label.setText(f"Files: {filtered}/{total}") + def _build_folder_tree(self): """Build folder tree from file paths (images + results).""" # Save current selection before rebuilding @@ -1138,7 +1224,7 @@ def _build_folder_tree(self): # Add all parent directories for parent in path.parents: parent_str = str(parent) - if parent_str != '.' and not parent_str.endswith('_results'): + if parent_str != "." and not parent_str.endswith("_results"): folders.add(parent_str) # Build tree structure @@ -1175,12 +1261,52 @@ def _build_folder_tree(self): if selected_folder is not None: self._restore_folder_selection(selected_folder, folder_items) + def _on_parameter_changed(self, param_name: str, value: object): + """Handle parameter changes from the tabbed form. + + Updates view button states when streaming config 'enabled' fields change. + """ + logger.info( + f"🔔 ImageBrowser._on_parameter_changed: param_name={param_name}, value={value}" + ) + + # Strip leading dot if present (root PFM with field_id='' emits paths like ".napari_streaming_config.enabled") + normalized_param = param_name.lstrip(".") + + # Check if this is an 'enabled' field for any streaming config + for viewer_type in self.view_buttons.keys(): + enabled_path = f"{viewer_type}.enabled" + logger.debug(f" Checking if {normalized_param} == {enabled_path}") + if normalized_param == enabled_path: + logger.info(f" ✅ Match! Updating button state for {viewer_type}") + # Update the corresponding view button state + self._update_view_button_state(viewer_type) + break + + def _update_view_button_state(self, viewer_type: str): + """Update a single view button's enabled state based on selection and config. + + Args: + viewer_type: The viewer type key (e.g., 'napari_streaming_config') + """ + if viewer_type not in self.view_buttons: + logger.warning(f" ⚠️ viewer_type {viewer_type} not in view_buttons") + return + + has_selection = len(self.image_table_browser.get_selected_keys()) > 0 + is_enabled = self._is_viewer_enabled(viewer_type) + logger.info( + f" 🔘 Updating button for {viewer_type}: has_selection={has_selection}, is_enabled={is_enabled}, final={has_selection and is_enabled}" + ) + self.view_buttons[viewer_type].setEnabled(has_selection and is_enabled) + def _on_files_selected(self, keys: list): """Handle selection change from ImageTableBrowser.""" has_selection = len(keys) > 0 - # Enable buttons based on selection AND checkbox state - self.view_napari_btn.setEnabled(has_selection and self.napari_enable_checkbox.isChecked()) - self.view_fiji_btn.setEnabled(has_selection and self.fiji_enable_checkbox.isChecked()) + # Enable buttons based on selection AND enabled state from ObjectState + for viewer_type, view_btn in self.view_buttons.items(): + is_enabled = self._is_viewer_enabled(viewer_type) + view_btn.setEnabled(has_selection and is_enabled) # Backward compatibility alias def on_selection_changed(self): @@ -1192,12 +1318,14 @@ def _on_file_double_clicked(self, key: str, item: dict): """Handle double-click from ImageTableBrowser.""" file_info = self.all_files[key] - # Check if this is a result file (has 'type' field) or an image - if 'type' in file_info: + # Check if this is a result file (ROI, CSV, JSON) or an image + # Result files are stored in result_full_paths, images are not + filename = file_info["filename"] + if filename in self.result_full_paths: # This is a result file (ROI, CSV, JSON) self._handle_result_double_click(file_info) else: - # This is an image file (no 'type' field) + # This is an image file self._handle_image_double_click() # Backward compatibility alias @@ -1210,100 +1338,76 @@ def on_file_double_clicked(self, row: int, column: int): def _handle_image_double_click(self): """Handle double-click on an image - stream to enabled viewer(s).""" - napari_enabled = self.napari_enable_checkbox.isChecked() - fiji_enabled = self.fiji_enable_checkbox.isChecked() + # Find all enabled viewers by querying ObjectState + enabled_viewers = self._get_enabled_viewers() # Stream to whichever viewer(s) are enabled - if napari_enabled and fiji_enabled: - # Both enabled - stream to both - self.view_selected_in_napari() - self.view_selected_in_fiji() - elif napari_enabled: - # Only Napari enabled - self.view_selected_in_napari() - elif fiji_enabled: - # Only Fiji enabled - self.view_selected_in_fiji() + if enabled_viewers: + for viewer_type in enabled_viewers: + self._view_selected_in_viewer(viewer_type) else: - # Neither enabled - show message + # No viewers enabled - show message QMessageBox.information( self, "No Viewer Enabled", - "Please enable Napari or Fiji streaming to view images." + "Please enable at least one viewer streaming to view images.", ) def _handle_result_double_click(self, file_info: dict): """Handle double-click on a result file - stream ROIs or display CSV.""" - filename = file_info['filename'] + filename = file_info["filename"] # Result files are populated in load_results() which stores both # all_results[filename] and result_full_paths[filename] together - must exist file_path = self.result_full_paths[filename] - file_type = file_info['type'] + file_type = file_info["type"] - if file_type == 'ROI': + if file_type == "ROI": # Stream ROI JSON to enabled viewer(s) self._stream_roi_file(file_path) - elif file_type == 'CSV': + elif file_type == "CSV": # Open CSV in system default application import subprocess - subprocess.run(['xdg-open', str(file_path)]) - elif file_type == 'JSON': + + subprocess.run(["xdg-open", str(file_path)]) + elif file_type == "JSON": # Open JSON in system default application import subprocess - subprocess.run(['xdg-open', str(file_path)]) - - def view_selected_in_napari(self): - """View all selected images in Napari as a batch (builds hyperstack).""" - selected_keys = self.image_table_browser.get_selected_keys() - if not selected_keys: - return - - # Separate ROI files from images - image_filenames = [k for k in selected_keys if not k.endswith('.roi.zip')] - roi_filenames = [k for k in selected_keys if k.endswith('.roi.zip')] - - try: - # Stream ROI files in a batch (get viewer once, stream all ROIs) - if roi_filenames: - plate_path = Path(self.orchestrator.plate_path) - self._stream_roi_batch_to_napari(roi_filenames, plate_path) - # Stream image files as a batch - if image_filenames: - self._load_and_stream_batch_to_napari(image_filenames) + subprocess.run(["xdg-open", str(file_path)]) - except Exception as e: - logger.error(f"Failed to view images in Napari: {e}") - QMessageBox.warning(self, "Error", f"Failed to view images in Napari: {e}") - - def view_selected_in_fiji(self): - """View all selected images in Fiji as a batch (builds hyperstack).""" + def _view_selected_in_viewer(self, viewer_type: str): + """View all selected images in the specified viewer as a batch (builds hyperstack).""" selected_keys = self.image_table_browser.get_selected_keys() if not selected_keys: return # Separate ROI files from images - image_filenames = [k for k in selected_keys if not k.endswith('.roi.zip')] - roi_filenames = [k for k in selected_keys if k.endswith('.roi.zip')] + image_filenames = [k for k in selected_keys if not k.endswith(".roi.zip")] + roi_filenames = [k for k in selected_keys if k.endswith(".roi.zip")] - logger.info(f"🎯 IMAGE BROWSER: User selected {len(image_filenames)} images and {len(roi_filenames)} ROI files to view in Fiji") - logger.info(f"🎯 IMAGE BROWSER: Image filenames: {image_filenames[:5]}{'...' if len(image_filenames) > 5 else ''}") + logger.info( + f"🎯 IMAGE BROWSER: User selected {len(image_filenames)} images and {len(roi_filenames)} ROI files to view in {viewer_type}" + ) + if image_filenames: + logger.info( + f"🎯 IMAGE BROWSER: Image filenames: {image_filenames[:5]}{'...' if len(image_filenames) > 5 else ''}" + ) if roi_filenames: logger.info(f"🎯 IMAGE BROWSER: ROI filenames: {roi_filenames}") - try: + from objectstate import spawn_thread_with_context + + def _view_async(): # Stream ROI files in a batch (get viewer once, stream all ROIs) if roi_filenames: plate_path = Path(self.orchestrator.plate_path) - self._stream_roi_batch_to_fiji(roi_filenames, plate_path) + self._stream_rois_to_viewer(roi_filenames, plate_path, viewer_type) # Stream image files as a batch if image_filenames: - self._load_and_stream_batch_to_fiji(image_filenames) + self._stream_images_to_viewer(image_filenames, viewer_type) - except Exception as e: - logger.error(f"Failed to view images in Fiji: {e}") - QMessageBox.warning(self, "Error", f"Failed to view images in Fiji: {e}") + spawn_thread_with_context(_view_async, name=f"view_{viewer_type}") def _prepare_streaming(self, viewer_type: str) -> tuple: """Prepare for streaming: resolve config, get viewer, get read backend. @@ -1316,37 +1420,16 @@ def _prepare_streaming(self, viewer_type: str) -> tuple: plate_path = Path(self.orchestrator.plate_path) # Resolve backend - from openhcs.config_framework.global_config import get_current_global_config - from openhcs.core.config import GlobalPipelineConfig - global_config = get_current_global_config(GlobalPipelineConfig) - - if global_config.vfs_config.read_backend != Backend.AUTO: - read_backend = global_config.vfs_config.read_backend.value - else: - read_backend = self.orchestrator.microscope_handler.get_primary_backend( - plate_path, self.orchestrator.filemanager - ) - - # Resolve streaming config - from openhcs.config_framework.context_manager import config_context - from openhcs.config_framework.lazy_factory import resolve_lazy_configurations_for_serialization - - if viewer_type == 'napari': - config_key = 'napari_config' - LazyConfigClass = LazyNapariStreamingConfig - else: - config_key = 'fiji_config' - LazyConfigClass = LazyFijiStreamingConfig - - current_values = self._get_config_values(config_key) - temp_config = LazyConfigClass(**current_values) + read_backend = self.orchestrator.microscope_handler.get_primary_backend( + plate_path, self.orchestrator.filemanager + ) - with config_context(self.orchestrator.pipeline_config): - with config_context(temp_config): - resolved_config = resolve_lazy_configurations_for_serialization(temp_config) + # Get fully resolved streaming config from ObjectState (includes inheritance) + # get_resolved_value now returns reconstructed dataclass with all sub-fields populated + config = self.state.get_resolved_value(viewer_type) - viewer = self.orchestrator.get_or_create_visualizer(resolved_config) - return viewer, plate_path, read_backend, resolved_config + viewer = self.orchestrator.get_or_create_visualizer(config) + return viewer, plate_path, read_backend, config def _stream_images_to_viewer(self, filenames: list, viewer_type: str): """Load and stream images to specified viewer type.""" @@ -1360,31 +1443,29 @@ def _stream_images_to_viewer(self, filenames: list, viewer_type: str): config=config, viewer_type=viewer_type, status_callback=self._status_update_signal.emit, - error_callback=lambda e: self._show_streaming_error(e) if viewer_type == 'napari' - else self._show_fiji_streaming_error(e), + error_callback=lambda e: self._show_streaming_error(e) + if viewer_type == "napari" + else self._show_fiji_streaming_error(e), ) logger.info(f"Streaming {len(filenames)} images to {viewer_type}...") - def _load_and_stream_batch_to_napari(self, filenames: list): - """Load multiple images and stream as batch to Napari (builds hyperstack).""" - self._stream_images_to_viewer(filenames, 'napari') - - def _load_and_stream_batch_to_fiji(self, filenames: list): - """Load multiple images and stream as batch to Fiji (builds hyperstack).""" - self._stream_images_to_viewer(filenames, 'fiji') - @pyqtSlot(str) def _show_streaming_error(self, error_msg: str): """Show streaming error in UI thread (called via QMetaObject.invokeMethod).""" - QMessageBox.warning(self, "Streaming Error", f"Failed to stream images to Napari: {error_msg}") + QMessageBox.warning( + self, "Streaming Error", f"Failed to stream images to Napari: {error_msg}" + ) @pyqtSlot(str) def _show_fiji_streaming_error(self, error_msg: str): """Show Fiji streaming error in UI thread.""" - QMessageBox.warning(self, "Streaming Error", f"Failed to stream images to Fiji: {error_msg}") - + QMessageBox.warning( + self, "Streaming Error", f"Failed to stream images to Fiji: {error_msg}" + ) - def _stream_rois_to_viewer(self, roi_filenames: list, plate_path: Path, viewer_type: str): + def _stream_rois_to_viewer( + self, roi_filenames: list, plate_path: Path, viewer_type: str + ): """Stream ROI files to specified viewer type.""" viewer, _, _, config = self._prepare_streaming(viewer_type) @@ -1395,20 +1476,12 @@ def _stream_rois_to_viewer(self, roi_filenames: list, plate_path: Path, viewer_t config=config, viewer_type=viewer_type, status_callback=self._status_update_signal.emit, - error_callback=lambda e: self._show_streaming_error(e) if viewer_type == 'napari' - else self._show_fiji_streaming_error(e), + error_callback=lambda e: self._show_streaming_error(e) + if viewer_type == "napari" + else self._show_fiji_streaming_error(e), ) logger.info(f"Streaming {len(roi_filenames)} ROI files to {viewer_type}...") - def _stream_roi_batch_to_napari(self, roi_filenames: list, plate_path: Path): - """Stream a batch of ROI files to Napari asynchronously (never blocks UI).""" - self._stream_rois_to_viewer(roi_filenames, plate_path, 'napari') - - def _stream_roi_batch_to_fiji(self, roi_filenames: list, plate_path: Path): - """Stream a batch of ROI files to Fiji asynchronously (never blocks UI).""" - self._stream_rois_to_viewer(roi_filenames, plate_path, 'fiji') - - def _display_csv_file(self, csv_path: Path): """Display CSV file in preview area.""" try: @@ -1441,7 +1514,7 @@ def _display_json_file(self, json_path: Path): import json # Read JSON - with open(json_path, 'r') as f: + with open(json_path, "r") as f: data = json.load(f) # Format as pretty JSON @@ -1520,7 +1593,9 @@ def _detach_plate_view(self): window_layout.addWidget(self.plate_view_widget) # Connect close event to reattach - self.plate_view_detached_window.closeEvent = lambda event: self._on_detached_window_closed(event) + self.plate_view_detached_window.closeEvent = ( + lambda event: self._on_detached_window_closed(event) + ) # Show window self.plate_view_detached_window.show() @@ -1566,6 +1641,7 @@ def _on_detached_window_closed(self, event): def _on_wells_selected(self, well_ids: Set[str]): """Handle well selection from plate view.""" + logger.info(f"[WELLS_SELECTED] Received {len(well_ids)} wells: {well_ids}") self.selected_wells = well_ids self._apply_combined_filters() @@ -1591,12 +1667,16 @@ def _update_plate_view(self): for well_id in well_ids: row, col = parser.extract_component_coordinates(well_id) # Convert row letter to index (A=1, B=2, etc.) - row_idx = sum((ord(c.upper()) - ord('A') + 1) * (26 ** i) - for i, c in enumerate(reversed(row))) + row_idx = sum( + (ord(c.upper()) - ord("A") + 1) * (26**i) + for i, c in enumerate(reversed(row)) + ) coord_to_well[(row_idx, int(col))] = well_id # Update plate view with well IDs, dimensions, and coordinate mapping - self.plate_view_widget.set_available_wells(well_ids, plate_dimensions, coord_to_well) + self.plate_view_widget.set_available_wells( + well_ids, plate_dimensions, coord_to_well + ) # Handle subdirectory selection current_folder = self._get_current_folder() @@ -1607,9 +1687,13 @@ def _matches_wells(self, filename: str, metadata: dict) -> bool: """Check if image matches selected wells.""" try: well_id = self._extract_well_id(metadata) - return well_id in self.selected_wells - except (KeyError, ValueError): + matches = well_id in self.selected_wells + if not matches: + logger.debug(f"[MATCH] Well {well_id} not in selected_wells") + return matches + except (KeyError, ValueError) as e: # Image has no well metadata, doesn't match well filter + logger.debug(f"[MATCH] No well metadata for {filename}: {e}") return False def _get_current_folder(self) -> Optional[str]: @@ -1686,7 +1770,7 @@ def _extract_well_id(self, metadata: dict) -> str: Raises KeyError if metadata missing 'well' component. """ # Well ID is a single component in metadata - return str(metadata['well']) + return str(metadata["well"]) def _detect_plate_dimensions(self, well_ids: Set[str]) -> tuple[int, int]: """ @@ -1712,8 +1796,10 @@ def _detect_plate_dimensions(self, well_ids: Set[str]) -> tuple[int, int]: # Convert row letters to indices (A=1, B=2, AA=27, etc.) row_indices = [ - sum((ord(c.upper()) - ord('A') + 1) * (26 ** i) - for i, c in enumerate(reversed(row))) + sum( + (ord(c.upper()) - ord("A") + 1) * (26**i) + for i, c in enumerate(reversed(row)) + ) for row in rows ] diff --git a/openhcs/pyqt_gui/widgets/pipeline_editor.py b/openhcs/pyqt_gui/widgets/pipeline_editor.py index df9027657..ad925905c 100644 --- a/openhcs/pyqt_gui/widgets/pipeline_editor.py +++ b/openhcs/pyqt_gui/widgets/pipeline_editor.py @@ -13,8 +13,14 @@ from pathlib import Path from PyQt6.QtWidgets import ( - QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QListWidget, - QListWidgetItem, QLabel, QSplitter + QWidget, + QVBoxLayout, + QHBoxLayout, + QPushButton, + QListWidget, + QListWidgetItem, + QLabel, + QSplitter, ) from PyQt6.QtCore import Qt, pyqtSignal, QTimer from PyQt6.QtGui import QFont, QColor @@ -25,6 +31,7 @@ from polystore.filemanager import FileManager from openhcs.core.steps.function_step import FunctionStep from openhcs.core.pipeline import Pipeline + # Mixin imports REMOVED - now in ABC (handle_selection_change_with_prevention, CrossWindowPreviewMixin) from pyqt_reactive.theming import StyleSheetGenerator from pyqt_reactive.widgets.shared.scope_visual_config import ListItemType @@ -36,7 +43,10 @@ # Import shared list widget components (single source of truth) from pyqt_reactive.core import ReorderableListWidget -from pyqt_reactive.widgets.shared.list_item_delegate import MultilinePreviewItemDelegate, StyledText +from pyqt_reactive.widgets.shared.list_item_delegate import ( + MultilinePreviewItemDelegate, + StyledText, +) from pyqt_reactive.widgets.editors.simple_code_editor import SimpleCodeEditorService from openhcs.config_framework.lazy_factory import PREVIEW_LABEL_REGISTRY from openhcs.core.config import ProcessingConfig @@ -49,7 +59,10 @@ from openhcs.pyqt_gui.windows.dual_editor_window import DualEditorWindow # Import ABC base class (Phase 4 migration) -from pyqt_reactive.widgets.shared.abstract_manager_widget import AbstractManagerWidget, ListItemFormat +from pyqt_reactive.widgets.shared.abstract_manager_widget import ( + AbstractManagerWidget, + ListItemFormat, +) from openhcs.utils.performance_monitor import timer @@ -86,39 +99,31 @@ class PipelineEditorWidget(AbstractManagerWidget): # Declarative item hooks (replaces 9 trivial method overrides) ITEM_HOOKS = { - 'id_accessor': ('attr', 'name'), # getattr(item, 'name', '') - 'backing_attr': 'pipeline_steps', # self.pipeline_steps - 'selection_attr': 'selected_step', # self.selected_step = ... - 'selection_signal': 'step_selected', # self.step_selected.emit(...) - 'selection_emit_id': False, # emit the full step object - 'selection_clear_value': None, # emit None when cleared - 'items_changed_signal': 'pipeline_changed', # self.pipeline_changed.emit(...) - 'preserve_selection_pred': lambda self: bool(self.pipeline_steps), - 'list_item_data': 'item', # store the step object - 'scope_item_type': ListItemType.STEP, - 'scope_id_builder': lambda item, idx, w: w._build_step_scope_id(item), + "id_accessor": ("attr", "name"), # getattr(item, 'name', '') + "backing_attr": "pipeline_steps", # self.pipeline_steps + "selection_attr": "selected_step", # self.selected_step = ... + "selection_signal": "step_selected", # self.step_selected.emit(...) + "selection_emit_id": False, # emit the full step object + "selection_clear_value": None, # emit None when cleared + "items_changed_signal": "pipeline_changed", # self.pipeline_changed.emit(...) + "preserve_selection_pred": lambda self: bool(self.pipeline_steps), + "list_item_data": "item", # store the step object + "scope_item_type": ListItemType.STEP, + "scope_id_builder": lambda item, idx, w: w._build_step_scope_id(item), } - # Declarative preview field configuration (processed automatically in ABC.__init__) - # Labels auto-discovered from PREVIEW_LABEL_REGISTRY (set via @global_pipeline_config(preview_label=...)) - PREVIEW_FIELD_CONFIGS = [ - 'napari_streaming_config', # preview_label='NAP' - 'fiji_streaming_config', # preview_label='FIJI' - 'step_materialization_config', # preview_label='MAT' - ] - # Declarative list item format (replaces imperative format_item_for_display logic) + # Config indicators (NAP, FIJI, MAT) are auto-discovered via always_viewable_fields LIST_ITEM_FORMAT = ListItemFormat( - first_line=('func',), # func= shown after step name + first_line=("func",), # func= shown after step name preview_line=( - 'processing_config.variable_components', - 'processing_config.group_by', - 'processing_config.input_source', + "processing_config.variable_components", + "processing_config.group_by", + "processing_config.input_source", ), - show_config_indicators=True, formatters={ - 'func': '_format_func_preview', # Method name for complex formatting - 'processing_config.input_source': '_format_input_source_preview', + "func": "_format_func_preview", # Method name for complex formatting + "processing_config.input_source": "_format_input_source_preview", }, ) @@ -145,7 +150,7 @@ def _format_func_preview(self, func, state=None) -> Optional[str]: func_names = [self._get_func_name(f) for f in func if f is not None] return f"func=[{', '.join(func_names)}]" elif callable(func): - func_name = getattr(func, '__name__', str(func)) + func_name = getattr(func, "__name__", str(func)) return f"func={func_name}" elif isinstance(func, dict): # Use orchestrator's metadata cache for key→name mapping if available @@ -155,7 +160,7 @@ def _format_func_preview(self, func, state=None) -> Optional[str]: # Get group_by from ObjectState to determine component type for metadata lookup group_by = None if state: - group_by = state.get_resolved_value('processing_config.group_by') + group_by = state.get_resolved_value("processing_config.group_by") # Build {display_name: func_name} entries entries = [] @@ -163,7 +168,9 @@ def _format_func_preview(self, func, state=None) -> Optional[str]: # Try to get display name from metadata cache using step's group_by display_name = None if group_by and metadata_cache: - display_name = metadata_cache.get_component_metadata(group_by, str(key)) + display_name = metadata_cache.get_component_metadata( + group_by, str(key) + ) if display_name is None: display_name = str(key) func_name = self._get_func_name(func[key]) @@ -175,7 +182,7 @@ def _get_func_name(self, func_entry) -> str: """Extract function name from various func entry formats.""" if isinstance(func_entry, tuple) and len(func_entry) >= 1: # (func, kwargs) pattern - return getattr(func_entry[0], '__name__', str(func_entry[0])) + return getattr(func_entry[0], "__name__", str(func_entry[0])) elif isinstance(func_entry, list) and func_entry: # Chain pattern - show first→last first = self._get_func_name(func_entry[0]) @@ -184,25 +191,28 @@ def _get_func_name(self, func_entry) -> str: return f"{first}→{last}" return first elif callable(func_entry): - return getattr(func_entry, '__name__', str(func_entry)) + return getattr(func_entry, "__name__", str(func_entry)) return str(func_entry) def _format_input_source_preview(self, input_source) -> Optional[str]: """Format input_source field (only show if not default).""" - source_name = getattr(input_source, 'name', str(input_source)) - if source_name != 'PREVIOUS_STEP': + source_name = getattr(input_source, "name", str(input_source)) + if source_name != "PREVIOUS_STEP": return f"input={source_name}" return None # Skip default value - - # Signals pipeline_changed = pyqtSignal(list) # List[FunctionStep] step_selected = pyqtSignal(object) # FunctionStep status_message = pyqtSignal(str) # status message - - def __init__(self, service_adapter, color_scheme: Optional[ColorScheme] = None, - gui_config: Optional[PyQtGUIConfig] = None, parent=None): + + def __init__( + self, + service_adapter, + color_scheme: Optional[ColorScheme] = None, + gui_config: Optional[PyQtGUIConfig] = None, + parent=None, + ): """ Initialize the pipeline editor widget. @@ -223,6 +233,9 @@ def __init__(self, service_adapter, color_scheme: Optional[ColorScheme] = None, # Note: orchestrator is looked up dynamically via _get_current_orchestrator() self.plate_manager = None + # Clipboard for copy-paste operations (in-memory only) + self._clipboard_steps: List[FunctionStep] = [] + # Initialize base class (creates style_generator, event_bus, item_list, buttons, status_label internally) # Also auto-processes PREVIEW_FIELD_CONFIGS declaratively super().__init__(service_adapter, color_scheme, gui_config, parent) @@ -244,10 +257,19 @@ def setup_connections(self): # Step-specific signal self.pipeline_changed.connect(self.on_pipeline_changed) + self._suppress_pipeline_state_sync = False + + # Keyboard shortcuts for copy-paste + from PyQt6.QtGui import QShortcut, QKeySequence + + QShortcut(QKeySequence("Ctrl+C"), self, self._action_copy_steps) + QShortcut(QKeySequence("Ctrl+V"), self, self._action_paste_steps) # ========== Pipeline ObjectState Management ========== - def _ensure_pipeline_state(self, plate_path: str) -> ObjectState: + def _ensure_pipeline_state( + self, plate_path: str, *, register: bool = True + ) -> ObjectState: """Get or create Pipeline ObjectState for a plate. Args: @@ -270,9 +292,10 @@ def _ensure_pipeline_state(self, plate_path: str) -> ObjectState: state = ObjectState( object_instance=pipeline, scope_id=pipeline_scope, - parent_state=ObjectStateRegistry.get_by_scope(plate_path) + parent_state=ObjectStateRegistry.get_by_scope(plate_path), ) - ObjectStateRegistry.register(state) + if register: + ObjectStateRegistry.register(state, _skip_snapshot=True) return state @@ -295,18 +318,20 @@ def _get_steps_from_pipeline_state(self, plate_path: str) -> List[FunctionStep]: for scope_id in step_scope_ids: step_state = ObjectStateRegistry.get_by_scope(scope_id) if step_state: - steps.append(step_state.object_instance) + steps.append(step_state.to_object()) return steps - def _update_pipeline_steps(self, plate_path: str, steps: List[FunctionStep]) -> None: + def _update_pipeline_steps( + self, plate_path: str, steps: List[FunctionStep] + ) -> None: """Update Pipeline ObjectState with new step list. Args: plate_path: Path of the plate steps: New list of FunctionStep objects """ - pipeline_state = self._ensure_pipeline_state(plate_path) + pipeline_state = self._ensure_pipeline_state(plate_path, register=False) if not pipeline_state: return @@ -315,22 +340,48 @@ def _update_pipeline_steps(self, plate_path: str, steps: List[FunctionStep]) -> # Build scope IDs and register each step with ObjectState step_scope_ids = [] + to_register: list[ObjectState] = [] for step in steps: scope_id = ScopeTokenService.build_scope_id(plate_path, step) step_scope_ids.append(scope_id) # Register step with ObjectState if not already registered existing = ObjectStateRegistry.get_by_scope(scope_id) - if not existing: + step_state = existing + if not step_state: state = ObjectState( object_instance=step, scope_id=scope_id, parent_state=ObjectStateRegistry.get_by_scope(plate_path), ) - ObjectStateRegistry.register(state) - logger.debug(f"Registered ObjectState for step: {scope_id}") - - # Update Pipeline ObjectState parameter + step_state = state + to_register.append(step_state) + + # Register function ObjectStates alongside the step + func_items = self._normalize_func_items(step.func) + for func_obj, kwargs in func_items: + func_scope_id = ScopeTokenService.build_scope_id(scope_id, func_obj) + if ObjectStateRegistry.get_by_scope(func_scope_id): + continue + reserved_param = self._get_reserved_param_name(func_obj) + exclude_params = [reserved_param] if reserved_param else None + func_state = ObjectState( + object_instance=func_obj, + scope_id=func_scope_id, + parent_state=step_state, + exclude_params=exclude_params, + initial_values=kwargs, + ) + to_register.append(func_state) + + # Register pipeline + steps + update step_scope_ids + # NOTE: This is called within an atomic block from the caller (delete/paste/add) + # Do NOT wrap in atomic() here - let the caller manage the atomic context + if ObjectStateRegistry.get_by_scope(pipeline_state.scope_id) is None: + ObjectStateRegistry.register(pipeline_state) + for state in to_register: + ObjectStateRegistry.register(state) + logger.debug(f"Registered ObjectState for step: {state.scope_id}") pipeline_state.update_parameter("step_scope_ids", step_scope_ids) @property @@ -346,7 +397,9 @@ def plate_pipelines(self) -> Dict[str, List[FunctionStep]]: """ # Prefer the plate root scope used by PlateManager (ROOT_SCOPE_ID="__plates__") # Fallback to global scope "" for legacy behavior. - root_state = ObjectStateRegistry.get_by_scope("__plates__") or ObjectStateRegistry.get_by_scope("") + root_state = ObjectStateRegistry.get_by_scope( + "__plates__" + ) or ObjectStateRegistry.get_by_scope("") if not root_state: return {} @@ -361,8 +414,10 @@ def plate_pipelines(self) -> Dict[str, List[FunctionStep]]: return result # ========== Business Logic Methods (Extracted from Textual) ========== - - def format_item_for_display(self, step: FunctionStep, live_context_snapshot=None) -> Tuple[str, str]: + + def format_item_for_display( + self, step: FunctionStep, live_context_snapshot=None + ) -> Tuple[str, str]: """ Format step for display in the list with constructor value preview. @@ -376,7 +431,7 @@ def format_item_for_display(self, step: FunctionStep, live_context_snapshot=None Returns: Tuple of (StyledText with segments, step_name) """ - step_name: str = getattr(step, 'name', 'Unknown Step') or 'Unknown Step' + step_name: str = getattr(step, "name", "Unknown Step") or "Unknown Step" # Use declarative format from LIST_ITEM_FORMAT styled = self._build_item_display_from_format( @@ -387,49 +442,55 @@ def format_item_for_display(self, step: FunctionStep, live_context_snapshot=None def _create_step_tooltip(self, step: FunctionStep) -> str: """Create detailed tooltip for a step showing all constructor values.""" - step_name = getattr(step, 'name', 'Unknown Step') + step_name = getattr(step, "name", "Unknown Step") tooltip_lines = [f"Step: {step_name}"] # Function details - func = getattr(step, 'func', None) + func = getattr(step, "func", None) if func: if isinstance(func, list): if len(func) == 1: - func_name = getattr(func[0], '__name__', str(func[0])) + func_name = getattr(func[0], "__name__", str(func[0])) tooltip_lines.append(f"Function: {func_name}") else: - func_names = [getattr(f, '__name__', str(f)) for f in func[:3]] + func_names = [getattr(f, "__name__", str(f)) for f in func[:3]] if len(func) > 3: - func_names.append(f"... +{len(func)-3} more") + func_names.append(f"... +{len(func) - 3} more") tooltip_lines.append(f"Functions: {', '.join(func_names)}") elif callable(func): - func_name = getattr(func, '__name__', str(func)) + func_name = getattr(func, "__name__", str(func)) tooltip_lines.append(f"Function: {func_name}") elif isinstance(func, dict): - tooltip_lines.append(f"Function: Dictionary with {len(func)} routing keys") + tooltip_lines.append( + f"Function: Dictionary with {len(func)} routing keys" + ) else: tooltip_lines.append("Function: None") # Variable components - var_components = getattr(step, 'variable_components', None) + var_components = getattr(step, "variable_components", None) if var_components: - comp_names = [getattr(c, 'name', str(c)) for c in var_components] + comp_names = [getattr(c, "name", str(c)) for c in var_components] tooltip_lines.append(f"Variable Components: [{', '.join(comp_names)}]") else: tooltip_lines.append("Variable Components: None") # Group by - group_by = getattr(step, 'group_by', None) + group_by = getattr(step, "group_by", None) if group_by and group_by.value is not None: # Check for GroupBy.NONE - group_name = getattr(group_by, 'name', str(group_by)) + group_name = getattr(group_by, "name", str(group_by)) tooltip_lines.append(f"Group By: {group_name}") else: tooltip_lines.append("Group By: None") # Input source (access from processing_config) - input_source = getattr(step.processing_config, 'input_source', None) if hasattr(step, 'processing_config') else None + input_source = ( + getattr(step.processing_config, "input_source", None) + if hasattr(step, "processing_config") + else None + ) if input_source: - source_name = getattr(input_source, 'name', str(input_source)) + source_name = getattr(input_source, "name", str(input_source)) tooltip_lines.append(f"Input Source: {source_name}") else: tooltip_lines.append("Input Source: None") @@ -440,15 +501,15 @@ def _create_step_tooltip(self, step: FunctionStep) -> str: # Helper to format config details based on type def format_config_detail(config_attr: str, config) -> str: """Format config detail string based on config type.""" - if config_attr == 'step_materialization_config': + if config_attr == "step_materialization_config": return "• Materialization Config: Enabled" - elif config_attr == 'napari_streaming_config': - port = getattr(config, 'port', 'default') + elif config_attr == "napari_streaming_config": + port = getattr(config, "port", "default") return f"• Napari Streaming: Port {port}" - elif config_attr == 'fiji_streaming_config': + elif config_attr == "fiji_streaming_config": return "• Fiji Streaming: Enabled" - elif config_attr == 'step_well_filter_config': - well_filter = getattr(config, 'well_filter', 'default') + elif config_attr == "step_well_filter_config": + well_filter = getattr(config, "well_filter", "default") return f"• Well Filter: {well_filter}" else: # Generic fallback for unknown config types @@ -468,7 +529,9 @@ def format_config_detail(config_attr: str, config) -> str: if not in_registry: continue # Check enabled field if exists - if is_dataclass(config) and 'enabled' in {ff.name for ff in fields(config)}: + if is_dataclass(config) and "enabled" in { + ff.name for ff in fields(config) + }: if not config.enabled: continue config_details.append(format_config_detail(f.name, config)) @@ -477,7 +540,7 @@ def format_config_detail(config_attr: str, config) -> str: tooltip_lines.append("") # Empty line separator tooltip_lines.extend(config_details) - return '\n'.join(tooltip_lines) + return "\n".join(tooltip_lines) def action_add_step(self): """Handle Add Step button (adapted from Textual version).""" @@ -488,7 +551,7 @@ def action_add_step(self): step_name = f"Step_{len(self.pipeline_steps) + 1}" new_step = FunctionStep( func=[], # Start with empty function list - name=step_name + name=step_name, ) plate_scope = self.current_plate or "no_plate" ScopeTokenService.ensure_token(plate_scope, new_step) @@ -501,7 +564,11 @@ def handle_save(edited_step): """Handle step save from editor.""" # Use atomic operation to coalesce all ObjectState changes into one undo step is_new = edited_step not in self.pipeline_steps - label = f"add step {edited_step.name}" if is_new else f"edit step {edited_step.name}" + label = ( + f"add step {edited_step.name}" + if is_new + else f"edit step {edited_step.name}" + ) with ObjectStateRegistry.atomic(label): # Check if step already exists in pipeline (for Shift+Click saves) @@ -518,7 +585,11 @@ def handle_save(edited_step): self._update_pipeline_steps(self.current_plate, self.pipeline_steps) self.update_item_list() - self.pipeline_changed.emit(self.pipeline_steps) + self._suppress_pipeline_state_sync = True + try: + self.pipeline_changed.emit(self.pipeline_steps) + finally: + self._suppress_pipeline_state_sync = False # Create and show editor dialog within the correct config context orchestrator = self._get_current_orchestrator() @@ -526,7 +597,9 @@ def handle_save(edited_step): # SIMPLIFIED: Orchestrator context is automatically available through type-based registry # No need for explicit context management - dual-axis resolver handles it automatically if not orchestrator: - logger.info("No orchestrator found for step editor context, This should not happen.") + logger.info( + "No orchestrator found for step editor context, This should not happen." + ) editor = DualEditorWindow( step_data=new_step, @@ -534,15 +607,19 @@ def handle_save(edited_step): on_save_callback=handle_save, orchestrator=orchestrator, gui_config=self.gui_config, - parent=self + parent=self, ) # Set original step for change detection editor.set_original_step_for_change_detection() # Connect orchestrator config changes to step editor for live placeholder updates # This ensures the step editor's placeholders update when pipeline config is saved - if self.plate_manager and hasattr(self.plate_manager, 'orchestrator_config_changed'): - self.plate_manager.orchestrator_config_changed.connect(editor.on_orchestrator_config_changed) + if self.plate_manager and hasattr( + self.plate_manager, "orchestrator_config_changed" + ): + self.plate_manager.orchestrator_config_changed.connect( + editor.on_orchestrator_config_changed + ) logger.debug("Connected orchestrator_config_changed signal to step editor") editor.show() @@ -568,14 +645,19 @@ def action_auto_load_pipeline(self): # Use ABC template for unified code execution (handles registration sync) self._handle_edited_code(python_code) - self.status_message.emit(f"Auto-loaded {len(self.pipeline_steps)} steps from basic_pipeline.py") + self.status_message.emit( + f"Auto-loaded {len(self.pipeline_steps)} steps from basic_pipeline.py" + ) except Exception as e: import traceback + logger.error(f"Failed to auto-load basic_pipeline.py: {e}") logger.error(f"Full traceback:\n{traceback.format_exc()}") - self.service_adapter.show_error_dialog(f"Failed to auto-load pipeline: {str(e)}") - + self.service_adapter.show_error_dialog( + f"Failed to auto-load pipeline: {str(e)}" + ) + def action_code_pipeline(self): """Handle Code Pipeline button - edit pipeline as Python code.""" logger.debug("Code button pressed - opening code editor") @@ -597,7 +679,10 @@ def action_code_pipeline(self): # Check if user wants external editor (check environment variable) import os - use_external = os.environ.get('OPENHCS_USE_EXTERNAL_EDITOR', '').lower() in ('1', 'true', 'yes') + + use_external = os.environ.get( + "OPENHCS_USE_EXTERNAL_EDITOR", "" + ).lower() in ("1", "true", "yes") # Launch editor with callback - uses ABC _handle_edited_code template editor_service.edit_code( @@ -605,47 +690,67 @@ def action_code_pipeline(self): title="Edit Pipeline Steps", callback=self._handle_edited_code, # ABC template method use_external=use_external, - code_type='pipeline', - code_data={'clean_mode': True} + code_type="pipeline", + code_data={"clean_mode": True}, ) except Exception as e: logger.error(f"Failed to open pipeline code editor: {e}") - self.service_adapter.show_error_dialog(f"Failed to open code editor: {str(e)}") + self.service_adapter.show_error_dialog( + f"Failed to open code editor: {str(e)}" + ) # === Code Execution Hooks (ABC _handle_edited_code template) === - def _handle_code_execution_error(self, code: str, error: Exception, namespace: dict) -> Optional[dict]: + def _handle_code_execution_error( + self, code: str, error: Exception, namespace: dict + ) -> Optional[dict]: """Handle old-format step constructors by retrying with migration patch.""" error_msg = str(error) - if "unexpected keyword argument" in error_msg and ("group_by" in error_msg or "variable_components" in error_msg): - logger.info(f"Detected old-format step constructor, retrying with migration patch: {error}") + if "unexpected keyword argument" in error_msg and ( + "group_by" in error_msg or "variable_components" in error_msg + ): + logger.info( + f"Detected old-format step constructor, retrying with migration patch: {error}" + ) new_namespace = {} - with self._patch_lazy_constructors(), patch_step_constructors_for_migration(): + with ( + self._patch_lazy_constructors(), + patch_step_constructors_for_migration(), + ): exec(code, new_namespace) return new_namespace return None # Re-raise error def _apply_executed_code(self, namespace: dict) -> bool: """Extract pipeline_steps from namespace and apply to widget state.""" - if 'pipeline_steps' not in namespace: + if "pipeline_steps" not in namespace: return False - new_pipeline_steps = namespace['pipeline_steps'] + new_pipeline_steps = namespace["pipeline_steps"] self.pipeline_steps = new_pipeline_steps - self._normalize_step_scope_tokens() + # Don't register here; _update_pipeline_steps handles atomic registration + self._normalize_step_scope_tokens(register=False) # Update Pipeline ObjectState with new step list if self.current_plate: self._update_pipeline_steps(self.current_plate, self.pipeline_steps) - logger.debug(f"Updated Pipeline ObjectState ({len(self.pipeline_steps)} steps) for plate: {self.current_plate}") + logger.debug( + f"Updated Pipeline ObjectState ({len(self.pipeline_steps)} steps) for plate: {self.current_plate}" + ) self.update_item_list() - self.pipeline_changed.emit(self.pipeline_steps) - self.status_message.emit(f"Pipeline updated with {len(new_pipeline_steps)} steps") + self._suppress_pipeline_state_sync = True + try: + self.pipeline_changed.emit(self.pipeline_steps) + finally: + self._suppress_pipeline_state_sync = False + self.status_message.emit( + f"Pipeline updated with {len(new_pipeline_steps)} steps" + ) # Broadcast to global event bus for ALL windows to receive - self._broadcast_to_event_bus('pipeline', new_pipeline_steps) + self._broadcast_to_event_bus("pipeline", new_pipeline_steps) return True def _get_code_missing_error_message(self) -> str: @@ -667,40 +772,50 @@ def load_pipeline_from_file(self, file_path: Path): if steps is not None: self.pipeline_steps = steps - self._normalize_step_scope_tokens() + # Don't register here; _update_pipeline_steps handles atomic registration + self._normalize_step_scope_tokens(register=False) # Update Pipeline ObjectState with loaded steps if self.current_plate: self._update_pipeline_steps(self.current_plate, self.pipeline_steps) - logger.debug(f"Updated Pipeline ObjectState ({len(self.pipeline_steps)} steps) for plate: {self.current_plate}") + logger.debug( + f"Updated Pipeline ObjectState ({len(self.pipeline_steps)} steps) for plate: {self.current_plate}" + ) self.update_item_list() - self.pipeline_changed.emit(self.pipeline_steps) - self.status_message.emit(f"Loaded {len(steps)} steps from {file_path.name}") + self._suppress_pipeline_state_sync = True + try: + self.pipeline_changed.emit(self.pipeline_steps) + finally: + self._suppress_pipeline_state_sync = False + self.status_message.emit( + f"Loaded {len(steps)} steps from {file_path.name}" + ) else: self.status_message.emit(f"Invalid pipeline format in {file_path.name}") except Exception as e: logger.error(f"Failed to load pipeline: {e}") self.service_adapter.show_error_dialog(f"Failed to load pipeline: {e}") - + def save_pipeline_to_file(self, file_path: Path): """ Save pipeline to file (extracted from Textual version). - + Args: file_path: Path to save pipeline """ try: import dill as pickle - with open(file_path, 'wb') as f: + + with open(file_path, "wb") as f: pickle.dump(list(self.pipeline_steps), f) self.status_message.emit(f"Saved pipeline to {file_path.name}") - + except Exception as e: logger.error(f"Failed to save pipeline: {e}") self.service_adapter.show_error_dialog(f"Failed to save pipeline: {e}") - + def save_pipeline_for_plate(self, plate_path: str, pipeline: List[FunctionStep]): """ Save pipeline for specific plate (extracted from Textual version). @@ -711,7 +826,13 @@ def save_pipeline_for_plate(self, plate_path: str, pipeline: List[FunctionStep]) """ self._update_pipeline_steps(plate_path, pipeline) logger.debug(f"Updated Pipeline ObjectState for plate: {plate_path}") - + + def get_pipeline_for_plate(self, plate_path: str) -> List[FunctionStep]: + """Return the current pipeline definition from Pipeline ObjectState.""" + if not plate_path: + return [] + return self._get_steps_from_pipeline_state(plate_path) + def set_current_plate(self, plate_path: str): """ Set current plate and load its pipeline (extracted from Textual version). @@ -731,12 +852,14 @@ def set_current_plate(self, plate_path: str): if plate_path: plate_pipeline = self._get_steps_from_pipeline_state(plate_path) self.pipeline_steps = plate_pipeline - logger.info(f" → Loaded {len(plate_pipeline)} steps for plate from Pipeline ObjectState") + logger.info( + f" → Loaded {len(plate_pipeline)} steps for plate from Pipeline ObjectState" + ) else: self.pipeline_steps = [] logger.info(f" → No plate selected, cleared pipeline") - self._normalize_step_scope_tokens() + self._normalize_step_scope_tokens(register=False) # CRITICAL: Force cleanup of flash subscriptions when switching plates # This ensures FlashElements don't point to stale QListWidgetItems @@ -764,7 +887,9 @@ def on_orchestrator_config_changed(self, plate_path: str, effective_config): """ # Only refresh if this is for the current plate if plate_path == self.current_plate: - logger.debug(f"Refreshing placeholders for orchestrator config change: {plate_path}") + logger.debug( + f"Refreshing placeholders for orchestrator config change: {plate_path}" + ) # SIMPLIFIED: Orchestrator context is automatically available through type-based registry # No need for explicit context management - dual-axis resolver handles it automatically @@ -772,7 +897,9 @@ def on_orchestrator_config_changed(self, plate_path: str, effective_config): if orchestrator: # Trigger refresh of any open configuration windows or step forms # The type-based registry ensures they resolve against the updated orchestrator config - logger.debug(f"Step forms will now resolve against updated orchestrator config for: {plate_path}") + logger.debug( + f"Step forms will now resolve against updated orchestrator config for: {plate_path}" + ) else: logger.debug(f"No orchestrator found for config refresh: {plate_path}") @@ -788,41 +915,109 @@ def _build_step_scope_id(self, step: FunctionStep) -> str: def _get_item_insert_index(self, item: Any, scope_key: str) -> Optional[int]: """Get correct position for step re-insertion during time-travel.""" # Token format is e.g. "functionstep_3" - parse index from it - token = getattr(item, '_scope_token', None) + token = getattr(item, "_scope_token", None) if token: - parts = token.rsplit('_', 1) + parts = token.rsplit("_", 1) if len(parts) == 2 and parts[1].isdigit(): return min(int(parts[1]), len(self.pipeline_steps)) return None - def _normalize_step_scope_tokens(self) -> None: + def _normalize_step_scope_tokens(self, register: bool = True) -> None: """Ensure all steps have tokens and are registered.""" plate_scope = self.current_plate or "no_plate" ScopeTokenService.seed_from_objects(plate_scope, self.pipeline_steps) + if not register: + return for step in self.pipeline_steps: self._register_step_state(step) + def _normalize_func_items(self, func_value) -> list[tuple[Callable, dict]]: + if not func_value: + return [] + from pyqt_reactive.services.pattern_data_manager import PatternDataManager + + if isinstance(func_value, dict): + items = [] + for channel_funcs in func_value.values(): + items.extend(self._normalize_func_items(channel_funcs)) + return items + if isinstance(func_value, list): + items = [] + for item in func_value: + func_obj, kwargs = PatternDataManager.extract_func_and_kwargs(item) + if func_obj: + items.append((func_obj, kwargs)) + return items + func_obj, kwargs = PatternDataManager.extract_func_and_kwargs(func_value) + return [(func_obj, kwargs)] if func_obj else [] + + def _get_reserved_param_name(self, func: Callable) -> Optional[str]: + try: + sig = inspect.signature(func) + except (TypeError, ValueError): + return None + for param_name, _param in sig.parameters.items(): + if param_name in ("self", "cls"): + continue + return param_name + return None + def _register_step_state(self, step: FunctionStep) -> None: """Register ObjectState for a step (creates if not exists).""" scope_id = self._build_step_scope_id(step) # Check if already registered existing = ObjectStateRegistry.get_by_scope(scope_id) - if existing: - return + step_state = existing # Get context (PipelineConfig from orchestrator) orchestrator = self._get_current_orchestrator() context_obj = orchestrator.pipeline_config if orchestrator else None - state = ObjectState( - object_instance=step, - scope_id=scope_id, - parent_state=ObjectStateRegistry.get_by_scope(str(self.current_plate)) if self.current_plate else None, - # func is hidden from ParameterFormManager via _ui_special_fields but included in ObjectState + parent_state = ( + ObjectStateRegistry.get_by_scope(str(self.current_plate)) + if self.current_plate + else None ) - ObjectStateRegistry.register(state) - logger.debug(f"Registered ObjectState for step: {scope_id}") + + # Determine which ObjectStates need registering + to_register: list[ObjectState] = [] + + if step_state is None: + step_state = ObjectState( + object_instance=step, + scope_id=scope_id, + parent_state=parent_state, + # func is hidden from ParameterFormManager via _ui_special_fields but included in ObjectState + ) + to_register.append(step_state) + + # Register function ObjectStates alongside the step (atomic) + func_items = self._normalize_func_items(step.func) + for func_obj, kwargs in func_items: + func_scope_id = ScopeTokenService.build_scope_id(scope_id, func_obj) + if ObjectStateRegistry.get_by_scope(func_scope_id): + continue + reserved_param = self._get_reserved_param_name(func_obj) + exclude_params = [reserved_param] if reserved_param else None + func_state = ObjectState( + object_instance=func_obj, + scope_id=func_scope_id, + parent_state=step_state, + exclude_params=exclude_params, + initial_values=kwargs, + ) + to_register.append(func_state) + + if not to_register: + return + + # NOTE: Registration should be atomic with the calling operation (paste/add) + # Do NOT wrap in atomic() here - let the caller manage the atomic context + for state in to_register: + ObjectStateRegistry.register(state) + + logger.debug(f"Registered ObjectState for step (and functions): {scope_id}") def _unregister_step_state(self, step: FunctionStep) -> None: """Unregister ObjectState for a step and all its nested functions.""" @@ -830,7 +1025,9 @@ def _unregister_step_state(self, step: FunctionStep) -> None: # Cascade unregister: step + all nested functions (prevents memory leak) count = ObjectStateRegistry.unregister_scope_and_descendants(scope_id) - logger.debug(f"Cascade unregistered {count} ObjectState(s) for deleted step: {scope_id}") + logger.debug( + f"Cascade unregistered {count} ObjectState(s) for deleted step: {scope_id}" + ) # _merge_with_live_values() DELETED - use _merge_with_live_values() from base class # _get_step_preview_instance() DELETED - ObjectState provides resolved values directly @@ -839,7 +1036,9 @@ def _handle_full_preview_refresh(self) -> None: """Refresh all step preview labels.""" self.update_item_list() - def _refresh_step_items_by_index(self, indices: Iterable[int], live_context_snapshot=None) -> None: + def _refresh_step_items_by_index( + self, indices: Iterable[int], live_context_snapshot=None + ) -> None: """Refresh specific step items by index. Uses ObjectState for values.""" if not indices or not self.current_plate: return @@ -878,8 +1077,10 @@ def update_button_states(self): self.buttons["auto_load_pipeline"].setEnabled(has_plate and is_initialized) self.buttons["del_step"].setEnabled(has_steps) self.buttons["edit_step"].setEnabled(has_steps and has_selection) - self.buttons["code_pipeline"].setEnabled(has_plate and is_initialized) # Same as add button - orchestrator init is sufficient - + self.buttons["code_pipeline"].setEnabled( + has_plate and is_initialized + ) # Same as add button - orchestrator init is sufficient + # Event handlers (update_status, on_selection_changed, on_item_double_clicked, on_steps_reordered) # DELETED - provided by AbstractManagerWidget base class # Step-specific behavior implemented via abstract hooks (see end of file) @@ -887,14 +1088,16 @@ def update_button_states(self): def on_pipeline_changed(self, steps: List[FunctionStep]): """ Handle pipeline changes. - + Args: steps: New pipeline steps """ + if self._suppress_pipeline_state_sync: + return # Save pipeline to current plate if one is selected if self.current_plate: self.save_pipeline_for_plate(self.current_plate, steps) - + logger.debug(f"Pipeline changed: {len(steps)} steps") def _is_current_plate_initialized(self) -> bool: @@ -902,39 +1105,58 @@ def _is_current_plate_initialized(self) -> bool: if not self.current_plate: return False - # Get plate manager from main window - main_window = self._find_main_window() - if not main_window: - return False + # Use plate_manager reference if available (set by main window during connection) + # This works for both embedded widgets and floating windows + if self.plate_manager: + from objectstate import ObjectStateRegistry + + orchestrator = ObjectStateRegistry.get_object(self.current_plate) + if orchestrator is None: + return False + + # Check if orchestrator is in an initialized state (mirrors Textual TUI logic) + is_initialized = orchestrator.state in [ + OrchestratorState.READY, + OrchestratorState.COMPILED, + OrchestratorState.COMPLETED, + OrchestratorState.COMPILE_FAILED, + OrchestratorState.EXEC_FAILED, + ] + logger.debug( + f"PipelineEditor: Plate {self.current_plate} orchestrator state: {orchestrator.state}, initialized: {is_initialized}" + ) + return is_initialized - # Get plate manager widget from floating windows - plate_manager_window = main_window.floating_windows.get("plate_manager") - if not plate_manager_window: - return False + # Fallback: Try to find plate manager via ServiceRegistry + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + from pyqt_reactive.services import ServiceRegistry - layout = plate_manager_window.layout() - if not layout or layout.count() == 0: + plate_manager = ServiceRegistry.get(PlateManagerWidget) + if not plate_manager: return False from objectstate import ObjectStateRegistry + orchestrator = ObjectStateRegistry.get_object(self.current_plate) if orchestrator is None: return False # Check if orchestrator is in an initialized state (mirrors Textual TUI logic) - return orchestrator.state in [OrchestratorState.READY, OrchestratorState.COMPILED, - OrchestratorState.COMPLETED, OrchestratorState.COMPILE_FAILED, - OrchestratorState.EXEC_FAILED] - - + return orchestrator.state in [ + OrchestratorState.READY, + OrchestratorState.COMPILED, + OrchestratorState.COMPLETED, + OrchestratorState.COMPILE_FAILED, + OrchestratorState.EXEC_FAILED, + ] def _get_current_orchestrator(self) -> Optional[PipelineOrchestrator]: """Get the orchestrator for the currently selected plate.""" if not self.current_plate: return None from objectstate import ObjectStateRegistry - return ObjectStateRegistry.get_object(self.current_plate) + return ObjectStateRegistry.get_object(self.current_plate) # _find_main_window() moved to AbstractManagerWidget @@ -949,9 +1171,11 @@ def on_config_changed(self, new_config: GlobalPipelineConfig): # CRITICAL FIX: Refresh all placeholders when global config changes # This ensures pipeline config editor shows updated inherited values - if hasattr(self, 'form_manager') and self.form_manager: + if hasattr(self, "form_manager") and self.form_manager: self.form_manager.refresh_placeholder_text() - logger.info("Refreshed pipeline config placeholders after global config change") + logger.info( + "Refreshed pipeline config placeholders after global config change" + ) # ========== Abstract Hook Implementations (AbstractManagerWidget ABC) ========== @@ -964,7 +1188,7 @@ def action_add(self) -> None: def _perform_delete(self, items: List[Any]) -> None: """Remove steps from backing list (required abstract method).""" # Build descriptive label for undo - step_names = [getattr(step, 'name', '?') for step in items] + step_names = [getattr(step, "name", "?") for step in items] label = f"delete step{'s' if len(items) > 1 else ''} {', '.join(step_names)}" with ObjectStateRegistry.atomic(label): @@ -974,14 +1198,16 @@ def _perform_delete(self, items: List[Any]) -> None: # Build set of steps to delete (by identity, not equality) steps_to_delete = set(id(step) for step in items) - self.pipeline_steps = [s for s in self.pipeline_steps if id(s) not in steps_to_delete] - self._normalize_step_scope_tokens() + self.pipeline_steps = [ + s for s in self.pipeline_steps if id(s) not in steps_to_delete + ] + self._normalize_step_scope_tokens(register=False) # Sync to Pipeline ObjectState if self.current_plate: self._update_pipeline_steps(self.current_plate, self.pipeline_steps) - if self.selected_step in [getattr(step, 'name', '') for step in items]: + if self.selected_step in [getattr(step, "name", "") for step in items]: self.selected_step = "" def _show_item_editor(self, item: Any) -> None: @@ -1003,7 +1229,9 @@ def handle_save(edited_step): if step is step_to_edit: # Transfer scope token from old to new step prefix = ScopeTokenService._get_prefix(step_to_edit) - ScopeTokenService.get_generator(plate_scope, prefix).transfer(step_to_edit, edited_step) + ScopeTokenService.get_generator(plate_scope, prefix).transfer( + step_to_edit, edited_step + ) self.pipeline_steps[i] = edited_step break @@ -1021,14 +1249,18 @@ def handle_save(edited_step): orchestrator=orchestrator, gui_config=self.gui_config, parent=self, - step_index=step_index # Pass actual position for border pattern + step_index=step_index, # Pass actual position for border pattern ) # Set original step for change detection editor.set_original_step_for_change_detection() # Connect orchestrator config changes to step editor for live placeholder updates - if self.plate_manager and hasattr(self.plate_manager, 'orchestrator_config_changed'): - self.plate_manager.orchestrator_config_changed.connect(editor.on_orchestrator_config_changed) + if self.plate_manager and hasattr( + self.plate_manager, "orchestrator_config_changed" + ): + self.plate_manager.orchestrator_config_changed.connect( + editor.on_orchestrator_config_changed + ) logger.debug("Connected orchestrator_config_changed signal to step editor") editor.show() @@ -1063,12 +1295,12 @@ def _pre_update_list(self) -> Any: ObjectState provides resolved values directly - no need to collect LiveContextSnapshot. Just ensure scope tokens are normalized. """ - self._normalize_step_scope_tokens() + self._normalize_step_scope_tokens(register=False) return None # ObjectState provides values, no context needed def _post_reorder(self) -> None: """Additional cleanup after reorder - normalize tokens and emit signal.""" - self._normalize_step_scope_tokens() + self._normalize_step_scope_tokens(register=False) # Sync to Pipeline ObjectState if self.current_plate: @@ -1076,15 +1308,17 @@ def _post_reorder(self) -> None: self.pipeline_changed.emit(self.pipeline_steps) # Broadcast to global event bus so open step editors update their colors - self._broadcast_to_event_bus('pipeline', self.pipeline_steps) + self._broadcast_to_event_bus("pipeline", self.pipeline_steps) # Record snapshot for time-travel (reordering is a significant state change) - ObjectStateRegistry.record_snapshot("reorder steps", scope_id=str(self.current_plate)) + ObjectStateRegistry.record_snapshot( + "reorder steps", scope_id=str(self.current_plate) + ) # === Config Resolution Hook (domain-specific) === def _get_scope_for_item(self, item: Any) -> str: """PipelineEditor: scope = plate::step_token.""" - scope = self._build_step_scope_id(item) or '' + scope = self._build_step_scope_id(item) or "" logger.debug(f"⚡ FLASH_DEBUG _get_scope_for_item: item={item}, scope={scope}") return scope @@ -1106,10 +1340,65 @@ def closeEvent(self, event): def _on_time_travel_complete(self, dirty_states, triggering_scope): """Refresh pipeline list after time travel to reflect restored step order.""" if self.current_plate: - self.pipeline_steps = self._get_steps_from_pipeline_state(self.current_plate) + self.pipeline_steps = self._get_steps_from_pipeline_state( + self.current_plate + ) else: self.pipeline_steps = [] - self._normalize_step_scope_tokens() + self._normalize_step_scope_tokens(register=False) self.update_item_list() self.update_button_states() + + def _action_copy_steps(self): + """Copy selected steps to clipboard (Ctrl+C).""" + selected_steps = self.get_selected_items() + if not selected_steps: + self.status_message.emit("No steps selected to copy") + return + + self._clipboard_steps = [copy.deepcopy(step) for step in selected_steps] + step_names = [getattr(step, "name", "?") for step in selected_steps] + self.status_message.emit( + f"Copied {len(selected_steps)} step(s): {', '.join(step_names)}" + ) + + def _action_paste_steps(self): + """Paste steps from clipboard after selected step (Ctrl+V).""" + if not self._clipboard_steps: + self.status_message.emit("Clipboard is empty") + return + + if not self.current_plate: + self.status_message.emit("No plate selected") + return + + # Calculate insert position: after last selected index, or at end if nothing selected + selected_indices = self.item_list.selectedIndexes() + if selected_indices: + insert_after_index = max(idx.row() for idx in selected_indices) + else: + insert_after_index = len(self.pipeline_steps) - 1 + + step_names = [getattr(step, "name", "?") for step in self._clipboard_steps] + label = f"paste {len(self._clipboard_steps)} step(s): {', '.join(step_names)}" + + with ObjectStateRegistry.atomic(label): + # Insert steps after the selected position + insert_position = insert_after_index + 1 + for i, step in enumerate(self._clipboard_steps): + # Ensure fresh scope token for the copied step + ScopeTokenService.ensure_token(self.current_plate, step) + # Register with ObjectState (handles flashing automatically) + self._register_step_state(step) + # Insert into pipeline + self.pipeline_steps.insert(insert_position + i, step) + + # Update Pipeline ObjectState + self._update_pipeline_steps(self.current_plate, self.pipeline_steps) + + self.update_item_list() + self.pipeline_changed.emit(self.pipeline_steps) + self.status_message.emit( + f"Pasted {len(self._clipboard_steps)} step(s) after position {insert_after_index + 1}" + ) diff --git a/openhcs/pyqt_gui/widgets/plate_manager.py b/openhcs/pyqt_gui/widgets/plate_manager.py index 6c0b78f51..02b56b850 100644 --- a/openhcs/pyqt_gui/widgets/plate_manager.py +++ b/openhcs/pyqt_gui/widgets/plate_manager.py @@ -17,7 +17,10 @@ from PyQt6.QtCore import Qt, pyqtSignal from openhcs.core.config import GlobalPipelineConfig, PipelineConfig -from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator, OrchestratorState +from openhcs.core.orchestrator.orchestrator import ( + PipelineOrchestrator, + OrchestratorState, +) from openhcs.core.path_cache import PathCacheKey from polystore.filemanager import FileManager from polystore.base import _create_storage_registry @@ -29,7 +32,7 @@ ) from openhcs.config_framework.global_config import ( set_global_config_for_editing, - get_current_global_config + get_current_global_config, ) from openhcs.config_framework.context_manager import config_context from openhcs.config_framework.object_state import ObjectState, ObjectStateRegistry @@ -38,16 +41,42 @@ from openhcs.core.xdg_paths import get_config_file_path import openhcs.serialization.pycodify_formatters # noqa: F401 from pycodify import Assignment, BlankLine, CodeBlock, generate_python_source -from openhcs.processing.backends.analysis.consolidate_analysis_results import consolidate_multi_plate_summaries +from openhcs.processing.backends.analysis.consolidate_analysis_results import ( + consolidate_multi_plate_summaries, +) from pyqt_reactive.theming import ColorScheme from openhcs.pyqt_gui.windows.config_window import ConfigWindow from openhcs.pyqt_gui.windows.plate_viewer_window import PlateViewerWindow from pyqt_reactive.widgets.editors.simple_code_editor import SimpleCodeEditorService -from pyqt_reactive.widgets.shared.abstract_manager_widget import AbstractManagerWidget, ListItemFormat +from pyqt_reactive.widgets.shared.abstract_manager_widget import ( + AbstractManagerWidget, + ListItemFormat, +) from pyqt_reactive.forms import ParameterFormManager -from openhcs.pyqt_gui.widgets.shared.services.zmq_execution_service import ZMQExecutionService -from openhcs.pyqt_gui.widgets.shared.services.compilation_service import CompilationService +from pyqt_reactive.services import ExecutionServerInfo +from openhcs.pyqt_gui.widgets.shared.services.batch_workflow_service import ( + BatchWorkflowService, +) +from openhcs.pyqt_gui.widgets.shared.services.plate_status_presenter import ( + PlateStatusPresenter, +) +from openhcs.pyqt_gui.widgets.shared.services.execution_state import ( + BUSY_MANAGER_STATES, + ExecutionBatchRuntime, + ManagerExecutionState, + STOP_PENDING_MANAGER_STATES, + TerminalExecutionStatus, + parse_terminal_status, + terminal_ui_policy, +) +from openhcs.pyqt_gui.widgets.shared.services.zmq_client_service import ( + ZMQClientService, +) from pyqt_reactive.widgets.shared.scope_visual_config import ListItemType +from openhcs.core.progress import registry +from openhcs.core.progress.projection import ( + ExecutionRuntimeProjection, +) logger = logging.getLogger(__name__) @@ -65,10 +94,12 @@ class PlateManagerWidget(AbstractManagerWidget): Uses CrossWindowPreviewMixin for reactive preview labels showing orchestrator config states (num_workers, well_filter, streaming configs, etc.). + + Auto-registers with ServiceRegistry for decoupled lookup by window factory. """ TITLE = "Plate Manager" - BUTTON_GRID_COLUMNS = 4 # 2x4 grid for 8 buttons + BUTTON_GRID_COLUMNS = 0 # Single row with all buttons ENABLE_STATUS_SCROLLING = True # Marquee animation for long status messages BUTTON_CONFIGS = [ ("Add", "add_plate", "Add new plate directory"), @@ -81,24 +112,30 @@ class PlateManagerWidget(AbstractManagerWidget): ("Viewer", "view_metadata", "View plate metadata"), ] ACTION_REGISTRY = { - "add_plate": "action_add", "del_plate": "action_delete", - "edit_config": "action_edit_config", "init_plate": "action_init_plate", - "compile_plate": "action_compile_plate", "code_plate": "action_code_plate", + "add_plate": "action_add", + "del_plate": "action_delete", + "edit_config": "action_edit_config", + "init_plate": "action_init_plate", + "compile_plate": "action_compile_plate", + "code_plate": "action_code_plate", "view_metadata": "action_view_metadata", } DYNAMIC_ACTIONS = {"run_plate": "_resolve_run_action"} ITEM_NAME_SINGULAR = "plate" ITEM_NAME_PLURAL = "plates" ITEM_HOOKS = { - 'id_accessor': 'path', 'backing_attr': 'plates', - 'selection_attr': 'selected_plate_path', 'selection_signal': 'plate_selected', - 'selection_emit_id': True, 'selection_clear_value': '', - 'items_changed_signal': None, 'list_item_data': 'item', - 'preserve_selection_pred': lambda self: bool(self.plates), - 'scope_item_type': ListItemType.ORCHESTRATOR, - 'scope_id_attr': 'path', + "id_accessor": "path", + "backing_attr": "plates", + "selection_attr": "selected_plate_path", + "selection_signal": "plate_selected", + "selection_emit_id": True, + "selection_clear_value": "", + "items_changed_signal": None, + "list_item_data": "item", + "preserve_selection_pred": lambda self: bool(self.plates), + "scope_item_type": ListItemType.ORCHESTRATOR, + "scope_id_attr": "path", } - # Signals plate_selected = pyqtSignal(str) status_message = pyqtSignal(str) @@ -115,9 +152,15 @@ class PlateManagerWidget(AbstractManagerWidget): execution_error = pyqtSignal(str) _execution_complete_signal = pyqtSignal(dict, str) _execution_error_signal = pyqtSignal(str) - - def __init__(self, service_adapter, color_scheme: Optional[ColorScheme] = None, - gui_config=None, parent=None): + _all_plates_completed_signal = pyqtSignal(int, int) + + def __init__( + self, + service_adapter, + color_scheme: Optional[ColorScheme] = None, + gui_config=None, + parent=None, + ): """ Initialize the plate manager widget. @@ -138,19 +181,32 @@ def __init__(self, service_adapter, color_scheme: Optional[ColorScheme] = None, self.selected_plate_path: str = "" self.plate_configs: Dict[str, Dict] = {} self.plate_compiled_data: Dict[str, tuple] = {} # Store compiled pipeline data - self.current_execution_id: Optional[str] = None # Track current execution ID for cancellation - self.execution_state = "idle" + self.current_execution_id: Optional[str] = ( + None # Track current execution ID for cancellation + ) + self.execution_state = ManagerExecutionState.IDLE # Track per-plate execution state self.plate_execution_ids: Dict[str, str] = {} # plate_path -> execution_id - self.plate_execution_states: Dict[str, str] = {} # plate_path -> "queued" | "running" | "completed" | "failed" - - # Extracted services (Phase 1, 2) - self._zmq_service = ZMQExecutionService(self, port=7777) - self._compilation_service = CompilationService(self) + self.execution_runtime = ExecutionBatchRuntime() + + # Use shared ExecutionProgressTracker singleton (same instance as ZMQ server browser) + # This ensures both UI components show the same progress data + self._progress_tracker = registry() + self.plate_progress: Dict[str, Dict] = {} # Deprecated, kept for compatibility + self.plate_init_pending = set() + self.plate_compile_pending = set() + self._runtime_progress_projection = ExecutionRuntimeProjection() + self._execution_server_info: ExecutionServerInfo | None = None + self._plate_status_presenter = PlateStatusPresenter() + + # Unified batch workflow service + self._zmq_client_service = ZMQClientService(port=7777) + self._batch_workflow_service = BatchWorkflowService( + self, client_service=self._zmq_client_service + ) # Initialize base class (creates style_generator, event_bus, item_list, buttons, status_label internally) - # Also auto-processes PREVIEW_FIELD_CONFIGS declaratively super().__init__(service_adapter, color_scheme, gui_config, parent) # Setup UI (after base and subclass state is ready) @@ -161,13 +217,18 @@ def __init__(self, service_adapter, color_scheme: Optional[ColorScheme] = None, # Connect internal signals for thread-safe completion handling self._execution_complete_signal.connect(self._on_execution_complete) self._execution_error_signal.connect(self._on_execution_error) + self._all_plates_completed_signal.connect( + self._finalize_all_plates_completed_ui + ) logger.debug("Plate manager widget initialized") def cleanup(self): """Cleanup resources before widget destruction.""" logger.info("🧹 Cleaning up PlateManagerWidget resources...") - self._zmq_service.disconnect() + self._batch_workflow_service.cleanup() + self._batch_workflow_service.disconnect() + logger.info("✅ PlateManagerWidget cleanup completed") def _on_time_travel_complete(self, dirty_states, triggering_scope): @@ -187,8 +248,12 @@ def _on_time_travel_complete(self, dirty_states, triggering_scope): # Log for debugging root_state = self._ensure_root_state() current_paths = set(root_state.parameters.get("orchestrator_scope_ids") or []) - initialized = sum(1 for p in current_paths if ObjectStateRegistry.get_object(p) is not None) - logger.info(f"🕰️ Time travel complete: {initialized}/{len(current_paths)} plates initialized") + initialized = sum( + 1 for p in current_paths if ObjectStateRegistry.get_object(p) is not None + ) + logger.info( + f"🕰️ Time travel complete: {initialized}/{len(current_paths)} plates initialized" + ) # Clear plate configs cache - force reload from ObjectState # (PipelineConfig is properly restored by ObjectState time travel) @@ -215,7 +280,7 @@ def _ensure_root_state(self) -> ObjectState: if not state: root = RootState() state = ObjectState(object_instance=root, scope_id=ROOT_SCOPE_ID) - ObjectStateRegistry.register(state) + ObjectStateRegistry.register(state, _skip_snapshot=True) return state @property @@ -228,75 +293,109 @@ def plates(self) -> List[Dict]: root_state = self._ensure_root_state() plate_paths = root_state.parameters.get("orchestrator_scope_ids") or [] - return [ - { - 'name': Path(path).name, - 'path': path - } - for path in plate_paths - ] + return [{"name": Path(path).name, "path": path} for path in plate_paths] # ExecutionHost interface - def emit_status(self, msg: str) -> None: self.status_message.emit(msg) - def emit_error(self, msg: str) -> None: self.execution_error.emit(msg) - def emit_orchestrator_state(self, plate_path: str, state: str) -> None: self.orchestrator_state_changed.emit(plate_path, state) - def emit_execution_complete(self, result: dict, plate_path: str) -> None: self._execution_complete_signal.emit(result, plate_path) - def emit_clear_logs(self) -> None: self.clear_subprocess_logs.emit() + def emit_status(self, msg: str) -> None: + self.status_message.emit(msg) + + def set_runtime_progress_projection( + self, projection: ExecutionRuntimeProjection + ) -> None: + self._runtime_progress_projection = projection + + def set_execution_server_info(self, info: ExecutionServerInfo | None) -> None: + self._execution_server_info = info + + def emit_error(self, msg: str) -> None: + self.execution_error.emit(msg) + + def emit_orchestrator_state(self, plate_path: str, state: str) -> None: + self.orchestrator_state_changed.emit(plate_path, state) + + def emit_execution_complete(self, result: dict, plate_path: str) -> None: + self._execution_complete_signal.emit(result, plate_path) + + def emit_clear_logs(self) -> None: + self.clear_subprocess_logs.emit() # CompilationHost interface - def emit_progress_started(self, count: int) -> None: self.progress_started.emit(count) - def emit_progress_updated(self, value: int) -> None: self.progress_updated.emit(value) - def emit_progress_finished(self) -> None: self.progress_finished.emit() - def emit_compilation_error(self, plate_name: str, error: str) -> None: self.compilation_error.emit(plate_name, error) - def get_pipeline_definition(self, plate_path: str) -> List: return self._get_current_pipeline_definition(plate_path) + def emit_progress_started(self, count: int) -> None: + self.progress_started.emit(count) + + def emit_progress_updated(self, value: int) -> None: + self.progress_updated.emit(value) + + def emit_progress_finished(self) -> None: + self.progress_finished.emit() + + def emit_compilation_error(self, plate_name: str, error: str) -> None: + self.compilation_error.emit(plate_name, error) + + def get_pipeline_definition(self, plate_path: str) -> List: + return self._get_current_pipeline_definition(plate_path) - def on_plate_completed(self, plate_path: str, status: str, result: dict) -> None: + def notify_plate_completed( + self, plate_path: str, status: str, result: dict + ) -> None: + if "status" not in result: + result["status"] = status self._execution_complete_signal.emit(result, plate_path) - def on_all_plates_completed(self, completed_count: int, failed_count: int) -> None: - self._zmq_service.disconnect() - self.execution_state = "idle" + def notify_all_plates_completed( + self, completed_count: int, failed_count: int + ) -> None: + self._all_plates_completed_signal.emit(completed_count, failed_count) + + def _finalize_all_plates_completed_ui( + self, completed_count: int, failed_count: int + ) -> None: + self._batch_workflow_service.disconnect_async() + self.execution_state = ManagerExecutionState.IDLE self.current_execution_id = None - if completed_count > 1 and self.global_config.analysis_consolidation_config.enabled: + if ( + completed_count > 1 + and self.global_config.analysis_consolidation_config.enabled + ): try: self._consolidate_multi_plate_results() - self.status_message.emit(f"All done: {completed_count} completed, {failed_count} failed. Global summary created.") + self.status_message.emit( + f"All done: {completed_count} completed, {failed_count} failed. Global summary created." + ) except Exception as e: logger.error(f"Failed to create global summary: {e}", exc_info=True) - self.status_message.emit(f"All done: {completed_count} completed, {failed_count} failed. Global summary failed.") + self.status_message.emit( + f"All done: {completed_count} completed, {failed_count} failed. Global summary failed." + ) else: - self.status_message.emit(f"All done: {completed_count} completed, {failed_count} failed") - self.update_button_states() - - # Declarative preview field configuration (processed automatically in ABC.__init__) - # Labels auto-discovered from PREVIEW_LABEL_REGISTRY (set via @global_pipeline_config(preview_label=...)) - PREVIEW_FIELD_CONFIGS = [ - 'napari_streaming_config', # preview_label='NAP' - 'fiji_streaming_config', # preview_label='FIJI' - 'step_materialization_config', # preview_label='MAT' - ] + self.status_message.emit( + f"All done: {completed_count} completed, {failed_count} failed" + ) + self.refresh_execution_ui() # Declarative list item format for PlateManager # The config source is orchestrator.pipeline_config # Field abbreviations are declared on config classes via @global_pipeline_config(field_abbreviations=...) + # Config indicators (NAP, FIJI, MAT) are auto-discovered via always_viewable_fields LIST_ITEM_FORMAT = ListItemFormat( first_line=(), # No fields on first line (just name) preview_line=( - 'num_workers', - 'vfs_config.materialization_backend', - 'path_planning_config.well_filter', - 'path_planning_config.output_dir_suffix', - 'path_planning_config.global_output_folder', + "num_workers", + "vfs_config.materialization_backend", + "path_planning_config.well_filter", + "path_planning_config.output_dir_suffix", + "path_planning_config.global_output_folder", ), - detail_line_field='path', # Show plate path as detail line - show_config_indicators=True, + detail_line_field="path", # Show plate path as detail line ) # ========== CrossWindowPreviewMixin Hooks ========== def _handle_full_preview_refresh(self) -> None: """Refresh all preview labels.""" - logger.info("🔄 PlateManager._handle_full_preview_refresh: refreshing preview labels") + logger.info( + "🔄 PlateManager._handle_full_preview_refresh: refreshing preview labels" + ) self.update_item_list() def _update_single_plate_item(self, plate_path: str): @@ -304,15 +403,17 @@ def _update_single_plate_item(self, plate_path: str): for i in range(self.item_list.count()): item = self.item_list.item(i) plate_data = item.data(Qt.ItemDataRole.UserRole) - if plate_data and plate_data.get('path') == plate_path: + if plate_data and plate_data.get("path") == plate_path: display_text = self._format_plate_item_with_preview_text(plate_data) item.setText(display_text) - self._set_item_styling_roles(item, display_text, plate_data) # ABC helper + self._set_item_styling_roles( + item, display_text, plate_data + ) # ABC helper break def format_item_for_display(self, item: Dict, live_ctx=None) -> Tuple[str, str]: """Format plate item for display with preview (required abstract method).""" - return (self._format_plate_item_with_preview_text(item), item['path']) + return (self._format_plate_item_with_preview_text(item), item["path"]) def _format_plate_item_with_preview_text(self, plate: Dict): """Format plate item with status and config preview labels. @@ -320,28 +421,49 @@ def _format_plate_item_with_preview_text(self, plate: Dict): Uses declarative LIST_ITEM_FORMAT with orchestrator.pipeline_config as config source. """ status_prefix = "" - orchestrator = ObjectStateRegistry.get_object(plate['path']) + plate_path = plate["path"] + plate_key = str(plate_path) + orchestrator = ObjectStateRegistry.get_object(plate_path) + terminal_status = self.execution_runtime.terminal_status(plate_key) + execution_id = self.plate_execution_ids.get(plate_key) + runtime_projection = self._runtime_progress_projection.get_plate( + plate_id=plate_key, + execution_id=execution_id, + ) + is_active_execution = self.execution_runtime.is_active(plate_key) or ( + runtime_projection is not None + ) - if orchestrator: - state_map = { - OrchestratorState.READY: "✓ Init", OrchestratorState.COMPILED: "✓ Compiled", - OrchestratorState.COMPLETED: "✅ Complete", OrchestratorState.INIT_FAILED: "❌ Init Failed", - OrchestratorState.COMPILE_FAILED: "❌ Compile Failed", OrchestratorState.EXEC_FAILED: "❌ Exec Failed", - } - if orchestrator.state == OrchestratorState.EXECUTING: - exec_state = self.plate_execution_states.get(plate['path']) - status_prefix = {"queued": "⏳ Queued", "running": "🔄 Running"}.get(exec_state, "🔄 Executing") - else: - status_prefix = state_map.get(orchestrator.state, "") + status_prefix = self._plate_status_presenter.build_status_prefix( + orchestrator_state=orchestrator.state if orchestrator else None, + is_init_pending=plate_key in self.plate_init_pending, + is_compile_pending=plate_key in self.plate_compile_pending, + is_execution_active=is_active_execution, + terminal_status=terminal_status, + queue_position=self._queued_execution_position_for_plate(plate_key), + runtime_projection=runtime_projection, + ) # Use declarative format with orchestrator.pipeline_config as introspection source return self._build_item_display_from_format( item=orchestrator, - item_name=plate['name'], + item_name=plate["name"], status_prefix=status_prefix, - detail_line=plate['path'], + detail_line=plate["path"], ) + def _queued_execution_position_for_plate(self, plate_id: str) -> Optional[int]: + """Return queue position from latest execution server snapshot for this plate.""" + server_info = self._execution_server_info + if server_info is None: + return None + + for queued in server_info.queued_execution_entries: + if queued.plate_id != plate_id: + continue + return queued.queue_position + return None + def setup_connections(self): """Setup signal/slot connections (base class + plate-specific).""" self._setup_connections() @@ -352,13 +474,14 @@ def setup_connections(self): self.compilation_error.connect(self._handle_compilation_error) self.initialization_error.connect(self._handle_initialization_error) self.execution_error.connect(self._handle_execution_error) - self._execution_complete_signal.connect(self._on_execution_complete) - self._execution_error_signal.connect(self._on_execution_error) def _resolve_run_action(self) -> str: - """Resolve run/stop action based on current state. - """ - return "action_stop_execution" if self.is_any_plate_running() else "action_run_plate" + """Resolve run/stop action based on current state.""" + return ( + "action_stop_execution" + if self.is_any_plate_running() + else "action_run_plate" + ) def _update_orchestrator_global_config(self, orchestrator, new_global_config): """Update orchestrator global config reference and rebuild pipeline config if needed.""" @@ -368,14 +491,18 @@ def _update_orchestrator_global_config(self, orchestrator, new_global_config): orchestrator.pipeline_config = rebuild_lazy_config_with_new_global_reference( current_config, new_global_config, GlobalPipelineConfig ) - logger.info(f"Rebuilt orchestrator-specific config for plate: {orchestrator.plate_path}") + logger.info( + f"Rebuilt orchestrator-specific config for plate: {orchestrator.plate_path}" + ) # NOTE: ObjectState now auto-detects delegate changes, so no manual sync needed. # When the orchestrator's ObjectState is next accessed, it will automatically # detect that pipeline_config has been replaced and re-extract parameters. effective_config = orchestrator.get_effective_config() - self.orchestrator_config_changed.emit(str(orchestrator.plate_path), effective_config) + self.orchestrator_config_changed.emit( + str(orchestrator.plate_path), effective_config + ) # ========== Business Logic Methods ========== @@ -386,12 +513,12 @@ def action_add_plate(self): cache_key=PathCacheKey.PLATE_IMPORT, title="Select Plate Directory", fallback_path=Path.home(), - allow_multiple=True + allow_multiple=True, ) if selected_paths: self.add_plate_callback(selected_paths) - + def add_plate_callback(self, selected_paths: List[Path]): """ Handle plate directory selection (extracted from Textual version). @@ -423,7 +550,7 @@ def add_plate_callback(self, selected_paths: List[Path]): continue # Create orchestrator immediately (in CREATED state, not initialized) - self._create_orchestrator_for_plate(plate_path) + orchestrator_state = self._create_orchestrator_for_plate(plate_path) # Add plate path to root ObjectState new_paths.append(plate_path) @@ -432,7 +559,9 @@ def add_plate_callback(self, selected_paths: List[Path]): # Update root ObjectState if any plates were added if added_plates: - root_state.update_parameter("orchestrator_scope_ids", new_paths) + # Atomic: register orchestrator(s) + update orchestrator_scope_ids together + with ObjectStateRegistry.atomic("register orchestrators"): + root_state.update_parameter("orchestrator_scope_ids", new_paths) self.update_item_list() # Select the last added plate to ensure pipeline assignment works correctly @@ -440,11 +569,13 @@ def add_plate_callback(self, selected_paths: List[Path]): self.selected_plate_path = last_added_path logger.info(f"🔔 EMITTING plate_selected signal for: {last_added_path}") self.plate_selected.emit(last_added_path) - self.status_message.emit(f"Added {len(added_plates)} plate(s): {', '.join(added_plates)}") + self.status_message.emit( + f"Added {len(added_plates)} plate(s): {', '.join(added_plates)}" + ) else: self.status_message.emit("No new plates added (duplicates skipped)") - def _create_orchestrator_for_plate(self, plate_path: str) -> PipelineOrchestrator: + def _create_orchestrator_for_plate(self, plate_path: str) -> ObjectState: """ Create an orchestrator for a plate (in CREATED state, not initialized). @@ -459,14 +590,13 @@ def _create_orchestrator_for_plate(self, plate_path: str) -> PipelineOrchestrato The created PipelineOrchestrator instance """ # Skip if orchestrator already exists - existing = ObjectStateRegistry.get_object(plate_path) - if existing: - return existing + existing_state = ObjectStateRegistry.get_by_scope(plate_path) + if existing_state: + return existing_state plate_registry = _create_storage_registry() orchestrator = PipelineOrchestrator( - plate_path=plate_path, - storage_registry=plate_registry + plate_path=plate_path, storage_registry=plate_registry ) # Apply any saved config (e.g., from code loading) @@ -484,10 +614,12 @@ def _create_orchestrator_for_plate(self, plate_path: str) -> PipelineOrchestrato ) ObjectStateRegistry.register(orchestrator_state) - self.orchestrator_state_changed.emit(plate_path, OrchestratorState.CREATED.value) + self.orchestrator_state_changed.emit( + plate_path, OrchestratorState.CREATED.value + ) logger.info(f"Created orchestrator for plate (CREATED state): {plate_path}") - return orchestrator + return orchestrator_state # action_delete_plate() REMOVED - now uses ABC's action_delete() template with _perform_delete() hook @@ -495,26 +627,26 @@ def _validate_plates_for_operation(self, plates, operation_type): """Unified functional validator for all plate operations with debug logging.""" def _validate_compile(p): - orch = ObjectStateRegistry.get_object(p['path']) + orch = ObjectStateRegistry.get_object(p["path"]) if not orch: return False, "no_orchestrator_initialized" - pipeline_steps = self._get_current_pipeline_definition(p['path']) + pipeline_steps = self._get_current_pipeline_definition(p["path"]) if not pipeline_steps: return False, "empty_pipeline_definition" return True, "ok" def _validate_run(p): - orch = ObjectStateRegistry.get_object(p['path']) + orch = ObjectStateRegistry.get_object(p["path"]) if not orch: return False, "no_orchestrator_initialized" - if orch.state not in ['COMPILED', 'COMPLETED']: + if orch.state not in ["COMPILED", "COMPLETED"]: return False, f"orchestrator_state_not_runnable:{orch.state}" return True, "ok" validators = { - 'init': lambda p: (True, "ok"), # Init can work on any plates - 'compile': _validate_compile, - 'run': _validate_run, + "init": lambda p: (True, "ok"), # Init can work on any plates + "compile": _validate_compile, + "run": _validate_run, } validator = validators.get(operation_type, lambda p: (True, "ok")) @@ -528,8 +660,8 @@ def _validate_run(p): logger.info( "PLATE_VALIDATION [%s] plate=%s name=%s reason=%s", operation_type, - plate.get('path'), - plate.get('name'), + plate.get("path"), + plate.get("name"), reason, ) return invalid @@ -546,11 +678,11 @@ async def action_init_plate(self): """ self._ensure_context() selected_items = self.get_selected_items() - self._validate_plates_for_operation(selected_items, 'init') + self._validate_plates_for_operation(selected_items, "init") self.progress_started.emit(len(selected_items)) async def init_single_plate(i, plate): - plate_path = plate['path'] + plate_path = str(plate["path"]) # Get existing orchestrator (created during add) or create if missing orchestrator = ObjectStateRegistry.get_object(plate_path) @@ -560,61 +692,109 @@ async def init_single_plate(i, plate): orchestrator = self._create_orchestrator_for_plate(plate_path) # Skip if already initialized - if orchestrator.state in [OrchestratorState.READY, OrchestratorState.COMPILED, OrchestratorState.COMPLETED]: - logger.info(f"Orchestrator already initialized for {plate_path}, skipping") + if orchestrator.state in [ + OrchestratorState.READY, + OrchestratorState.COMPILED, + OrchestratorState.COMPLETED, + ]: + logger.info( + f"Orchestrator already initialized for {plate_path}, skipping" + ) self.progress_updated.emit(i + 1) return + self.plate_init_pending.add(plate_path) + self.update_item_list() + def do_init(): self._ensure_context() return orchestrator.initialize() try: await asyncio.get_event_loop().run_in_executor(None, do_init) + self.plate_init_pending.remove(plate_path) + self.update_item_list() self.orchestrator_state_changed.emit(plate_path, "READY") # If this plate is currently selected, emit signal to update pipeline editor # This ensures pipeline editor gets notified when the selected plate is initialized if plate_path == self.selected_plate_path: - logger.info(f"🔔 EMITTING plate_selected after init for currently selected plate: {plate_path}") + logger.info( + f"🔔 EMITTING plate_selected after init for currently selected plate: {plate_path}" + ) self.plate_selected.emit(plate_path) elif not self.selected_plate_path: # If no plate is selected, select this one self.selected_plate_path = plate_path - logger.info(f"🔔 EMITTING plate_selected after init (auto-selecting): {plate_path}") + logger.info( + f"🔔 EMITTING plate_selected after init (auto-selecting): {plate_path}" + ) self.plate_selected.emit(plate_path) except Exception as e: - logger.error(f"Failed to initialize plate {plate_path}: {e}", exc_info=True) + logger.error( + f"Failed to initialize plate {plate_path}: {e}", exc_info=True + ) orchestrator._state = OrchestratorState.INIT_FAILED - self.orchestrator_state_changed.emit(plate_path, OrchestratorState.INIT_FAILED.value) - self.initialization_error.emit(plate['name'], str(e)) + self.plate_init_pending.remove(plate_path) + self.update_item_list() + self.orchestrator_state_changed.emit( + plate_path, OrchestratorState.INIT_FAILED.value + ) + self.initialization_error.emit(plate["name"], str(e)) self.progress_updated.emit(i + 1) - await asyncio.gather(*[init_single_plate(i, p) for i, p in enumerate(selected_items)]) + await asyncio.gather( + *[init_single_plate(i, p) for i, p in enumerate(selected_items)] + ) self.progress_finished.emit() # Count successes and failures - success_count = len([p for p in selected_items if ObjectStateRegistry.get_object(p['path']) and ObjectStateRegistry.get_object(p['path']).state == OrchestratorState.READY]) - error_count = len([p for p in selected_items if ObjectStateRegistry.get_object(p['path']) and ObjectStateRegistry.get_object(p['path']).state == OrchestratorState.INIT_FAILED]) + success_count = len( + [ + p + for p in selected_items + if ObjectStateRegistry.get_object(p["path"]) + and ObjectStateRegistry.get_object(p["path"]).state + == OrchestratorState.READY + ] + ) + error_count = len( + [ + p + for p in selected_items + if ObjectStateRegistry.get_object(p["path"]) + and ObjectStateRegistry.get_object(p["path"]).state + == OrchestratorState.INIT_FAILED + ] + ) - msg = f"Successfully initialized {success_count} plate(s)" if error_count == 0 else f"Initialized {success_count} plate(s), {error_count} error(s)" + msg = ( + f"Successfully initialized {success_count} plate(s)" + if error_count == 0 + else f"Initialized {success_count} plate(s), {error_count} error(s)" + ) self.status_message.emit(msg) def action_edit_config(self): """Handle Edit Config button - per-orchestrator PipelineConfig editing.""" selected_items = self.get_selected_items() if not selected_items: - self.service_adapter.show_error_dialog("No plates selected for configuration.") + self.service_adapter.show_error_dialog( + "No plates selected for configuration." + ) return selected_orchestrators = [ - ObjectStateRegistry.get_object(item['path']) for item in selected_items - if ObjectStateRegistry.get_object(item['path']) is not None + ObjectStateRegistry.get_object(item["path"]) + for item in selected_items + if ObjectStateRegistry.get_object(item["path"]) is not None ] if not selected_orchestrators: - self.service_adapter.show_error_dialog("No initialized orchestrators selected.") + self.service_adapter.show_error_dialog( + "No initialized orchestrators selected." + ) return representative_orchestrator = selected_orchestrators[0] @@ -633,11 +813,17 @@ def handle_config_save(new_config: PipelineConfig) -> None: orchestrator.apply_pipeline_config(new_config) # Emit signal for UI components to refresh effective_config = orchestrator.get_effective_config() - self.orchestrator_config_changed.emit(str(orchestrator.plate_path), effective_config) + self.orchestrator_config_changed.emit( + str(orchestrator.plate_path), effective_config + ) # Auto-sync handles context restoration automatically when pipeline_config is accessed - if self.selected_plate_path and ObjectStateRegistry.get_object(self.selected_plate_path): - logger.debug(f"Orchestrator context automatically maintained after config save: {self.selected_plate_path}") + if self.selected_plate_path and ObjectStateRegistry.get_object( + self.selected_plate_path + ): + logger.debug( + f"Orchestrator context automatically maintained after config save: {self.selected_plate_path}" + ) count = len(selected_orchestrators) # Success message dialog removed for test automation compatibility @@ -648,10 +834,12 @@ def handle_config_save(new_config: PipelineConfig) -> None: config_class=PipelineConfig, current_config=current_plate_config, on_save_callback=handle_config_save, - orchestrator=representative_orchestrator # Pass orchestrator for context persistence + orchestrator=representative_orchestrator, # Pass orchestrator for context persistence ) - def _open_config_window(self, config_class, current_config, on_save_callback, orchestrator=None): + def _open_config_window( + self, config_class, current_config, on_save_callback, orchestrator=None + ): """Open configuration window with specified config class and current config. Singleton-per-scope behavior is handled automatically by BaseFormDialog.show(). @@ -667,8 +855,12 @@ def _open_config_window(self, config_class, current_config, on_save_callback, or scope_id = None config_window = ConfigWindow( - config_class, current_config, on_save_callback, - self.color_scheme, self, scope_id=scope_id + config_class, + current_config, + on_save_callback, + self.color_scheme, + self, + scope_id=scope_id, ) # BaseFormDialog.show() handles singleton-per-scope automatically config_window.show() @@ -677,7 +869,9 @@ def _open_config_window(self, config_class, current_config, on_save_callback, or def action_edit_global_config(self): """Handle global configuration editing - affects all orchestrators.""" - current_global_config = self.service_adapter.get_global_config() or GlobalPipelineConfig() + current_global_config = ( + self.service_adapter.get_global_config() or GlobalPipelineConfig() + ) def handle_global_config_save(new_config: GlobalPipelineConfig) -> None: self.service_adapter.set_global_config(new_config) @@ -689,15 +883,17 @@ def handle_global_config_save(new_config: GlobalPipelineConfig) -> None: self.global_config = new_config for plate in self.plates: - orchestrator = ObjectStateRegistry.get_object(plate['path']) + orchestrator = ObjectStateRegistry.get_object(plate["path"]) if orchestrator: self._update_orchestrator_global_config(orchestrator, new_config) - self.service_adapter.show_info_dialog("Global configuration applied to all orchestrators") + self.service_adapter.show_info_dialog( + "Global configuration applied to all orchestrators" + ) self._open_config_window( config_class=GlobalPipelineConfig, current_config=current_global_config, - on_save_callback=handle_global_config_save + on_save_callback=handle_global_config_save, ) def _save_global_config_to_cache(self, config: GlobalPipelineConfig): @@ -709,7 +905,9 @@ def _save_global_config_to_cache(self, config: GlobalPipelineConfig): if success: logger.info("Global config saved to cache for session persistence") else: - logger.error("Failed to save global config to cache - sync save returned False") + logger.error( + "Failed to save global config to cache - sync save returned False" + ) except Exception as e: logger.error(f"Failed to save global config to cache: {e}") # Don't show error dialog as this is not critical for immediate functionality @@ -723,21 +921,23 @@ async def action_compile_plate(self): return # Unified validation using functional validator - invalid_plates = self._validate_plates_for_operation(selected_items, 'compile') + invalid_plates = self._validate_plates_for_operation(selected_items, "compile") # Let validation failures bubble up as status messages if invalid_plates: - invalid_names = [p['name'] for p in invalid_plates] + invalid_names = [p["name"] for p in invalid_plates] logger.info( "PLATE_VALIDATION [compile] blocked %d plate(s): %s", len(invalid_names), ", ".join(invalid_names), ) - self.status_message.emit(f"Cannot compile invalid plates: {', '.join(invalid_names)}") + self.status_message.emit( + f"Cannot compile invalid plates: {', '.join(invalid_names)}" + ) return # Delegate to compilation service - await self._compilation_service.compile_plates(selected_items) + await self._batch_workflow_service.compile_plates(selected_items) async def action_run_plate(self): """Handle Run Plate button - execute compiled plates using ZMQ.""" @@ -746,55 +946,144 @@ async def action_run_plate(self): self.execution_error.emit("No plates selected to run.") return - ready_items = [item for item in selected_items if item.get('path') in self.plate_compiled_data] + ready_items = [ + item + for item in selected_items + if item.get("path") in self.plate_compiled_data + ] if not ready_items: - self.execution_error.emit("Selected plates are not compiled. Please compile first.") + self.execution_error.emit( + "Selected plates are not compiled. Please compile first." + ) return - await self._zmq_service.run_plates(ready_items) + await self._batch_workflow_service.run_plates(ready_items) + + def _maybe_auto_add_output_plate_orchestrator( + self, source_plate_path: str, result: dict + ) -> None: + """Optionally add the computed output plate root as a new orchestrator. + + The ZMQ execution server attaches `output_plate_root` to the completion + payload (computed by the compiler/path planner). If enabled via global + config, we add that path to Plate Manager when the run completes. + """ + auto_add_value = (result or {}).get("auto_add_output_plate_to_plate_manager") + if auto_add_value is None: + raise RuntimeError( + "Missing auto-add flag in completion result; expected from compile context." + ) + + auto_add = bool(auto_add_value) + + if not auto_add: + return + + output_plate_root = (result or {}).get("output_plate_root") + if not output_plate_root: + return + + output_plate_root = str(output_plate_root) + + root_state = self._ensure_root_state() + current_paths = root_state.parameters.get("orchestrator_scope_ids") or [] + if output_plate_root in current_paths: + return + + # PipelineOrchestrator requires a real directory for non-OMERO paths. + # Ensure it exists so we can register an orchestrator. + if not output_plate_root.startswith("/omero/"): + out_path = Path(output_plate_root) + try: + out_path.mkdir(parents=True, exist_ok=True) + except Exception as e: + raise RuntimeError( + f"Auto-add output plate skipped (mkdir failed): {output_plate_root} ({e})" + ) + + if not out_path.is_dir(): + raise RuntimeError( + f"Auto-add output plate skipped (not a dir): {output_plate_root}" + ) + + # Create orchestrator and add to root scope list (do not change selection) + self._create_orchestrator_for_plate(output_plate_root) + new_paths = list(current_paths) + new_paths.append(output_plate_root) + + with ObjectStateRegistry.atomic("auto-add output plate"): + root_state.update_parameter("orchestrator_scope_ids", new_paths) + + self.update_item_list() + logger.info( + "Auto-added output plate orchestrator: %s (from %s)", + output_plate_root, + source_plate_path, + ) def _on_execution_complete(self, result, plate_path): """Handle execution completion for a single plate (called from main thread via signal).""" - try: - status = result.get('status') - logger.info(f"Plate {plate_path} completed with status: {status}") - - # Update plate state and orchestrator - if status == 'complete': - self.plate_execution_states[plate_path] = "completed" - self.status_message.emit(f"✓ Completed {plate_path}") - new_state = OrchestratorState.COMPLETED - elif status == 'cancelled': - self.plate_execution_states[plate_path] = "failed" - self.status_message.emit(f"✗ Cancelled {plate_path}") - new_state = OrchestratorState.READY - else: - self.plate_execution_states[plate_path] = "failed" - error_msg = result.get('message', 'Unknown error') - traceback_str = result.get('traceback', '') + status = parse_terminal_status(result.get("status")) + logger.info("Plate %s completed with status: %s", plate_path, status.value) - # Include traceback in error message if available - if traceback_str: - full_error = f"Execution failed for {plate_path}:\n\n{traceback_str}" - else: - full_error = f"Execution failed for {plate_path}: {error_msg}" + self.plate_progress.pop(plate_path, None) - self.execution_error.emit(full_error) - new_state = OrchestratorState.EXEC_FAILED + policy = terminal_ui_policy(status) + self.execution_runtime.mark_terminal(plate_path, status) - orchestrator = ObjectStateRegistry.get_object(plate_path) - if orchestrator: - orchestrator._state = new_state - self.orchestrator_state_changed.emit(plate_path, new_state.value) + if policy.status_prefix: + self.status_message.emit(f"{policy.status_prefix} {plate_path}") + if policy.emit_failure: + self.execution_error.emit( + self._build_execution_failure_message(plate_path, result) + ) + if ( + policy.auto_add_output_plate + and self.execution_state == ManagerExecutionState.RUNNING + ): + self._maybe_auto_add_output_plate_orchestrator(plate_path, result) + elif policy.auto_add_output_plate: + logger.info( + "Skipping auto-add output plate (execution_state=%s)", + self.execution_state, + ) - # Reset execution state to idle and update button states - # This ensures Stop/Force Kill button returns to "Run" state - self.execution_state = "idle" - self.current_execution_id = None - self.update_button_states() + new_state = policy.orchestrator_state - except Exception as e: - logger.error(f"Error handling execution completion: {e}", exc_info=True) + orchestrator = ObjectStateRegistry.get_object(plate_path) + if orchestrator: + orchestrator._state = new_state + self.orchestrator_state_changed.emit(plate_path, new_state.value) + + self.clear_plate_execution_tracking(plate_path, clear_terminal=False) + self._maybe_reset_execution_state_after_stop() + self.refresh_execution_ui() + + @staticmethod + def _build_execution_failure_message(plate_path: str, result: dict) -> str: + traceback_str = result.get("traceback", "") + if traceback_str: + return f"Execution failed for {plate_path}:\n\n{traceback_str}" + error_msg = result.get("message", "Unknown error") + return f"Execution failed for {plate_path}: {error_msg}" + + def _maybe_reset_execution_state_after_stop(self) -> None: + """Reset run/stop UI once all plates are terminal after a stop request.""" + if self.execution_state not in STOP_PENDING_MANAGER_STATES: + return + + if not self.execution_runtime.all_batch_terminal(): + return + + server_info = self._execution_server_info + if server_info is not None and ( + server_info.running_execution_entries + or server_info.queued_execution_entries + ): + return + + self.execution_state = ManagerExecutionState.IDLE + self.current_execution_id = None def _consolidate_multi_plate_results(self): """Consolidate results from multiple completed plates into a global summary.""" @@ -802,43 +1091,66 @@ def _consolidate_multi_plate_results(self): path_config = self.global_config.path_planning_config analysis_config = self.global_config.analysis_consolidation_config - for plate_path_str, state in self.plate_execution_states.items(): - if state != "completed": + for ( + plate_path_str, + terminal_status, + ) in self.execution_runtime.terminal_status_by_plate.items(): + if terminal_status != TerminalExecutionStatus.COMPLETE: continue plate_path = Path(plate_path_str) - base = Path(path_config.global_output_folder) if path_config.global_output_folder else plate_path.parent - output_plate_root = base / f"{plate_path.name}{path_config.output_dir_suffix}" + base = ( + Path(path_config.global_output_folder) + if path_config.global_output_folder + else plate_path.parent + ) + output_plate_root = ( + base / f"{plate_path.name}{path_config.output_dir_suffix}" + ) materialization_path = self.global_config.materialization_results_path - results_dir = Path(materialization_path) if Path(materialization_path).is_absolute() else output_plate_root / materialization_path + results_dir = ( + Path(materialization_path) + if Path(materialization_path).is_absolute() + else output_plate_root / materialization_path + ) summary_path = results_dir / analysis_config.output_filename if summary_path.exists(): summary_paths.append(str(summary_path)) plate_names.append(output_plate_root.name) else: - logger.warning(f"No summary found for plate {plate_path} at {summary_path}") + logger.warning( + f"No summary found for plate {plate_path} at {summary_path}" + ) if len(summary_paths) < 2: return - global_output_dir = Path(path_config.global_output_folder) if path_config.global_output_folder else Path(summary_paths[0]).parent.parent.parent - global_summary_path = global_output_dir / analysis_config.global_summary_filename + global_output_dir = ( + Path(path_config.global_output_folder) + if path_config.global_output_folder + else Path(summary_paths[0]).parent.parent.parent + ) + global_summary_path = ( + global_output_dir / analysis_config.global_summary_filename + ) - logger.info(f"Consolidating {len(summary_paths)} summaries to {global_summary_path}") + logger.info( + f"Consolidating {len(summary_paths)} summaries to {global_summary_path}" + ) consolidate_multi_plate_summaries( summary_paths=summary_paths, output_path=str(global_summary_path), - plate_names=plate_names + plate_names=plate_names, ) logger.info(f"✅ Global summary created: {global_summary_path}") def _on_execution_error(self, error_msg): """Handle execution error (called from main thread via signal).""" self.execution_error.emit(f"Execution error: {error_msg}") - self.execution_state = "idle" + self.execution_state = ManagerExecutionState.IDLE self.current_execution_id = None - self.update_button_states() + self.refresh_execution_ui() def action_stop_execution(self): """Handle Stop Execution via ZMQ. @@ -848,21 +1160,26 @@ def action_stop_execution(self): """ logger.info("🛑 action_stop_execution CALLED") - if self._zmq_service.zmq_client is None: - logger.warning("No active ZMQ execution to stop") - return - is_force_kill = self.buttons["run_plate"].text() == "Force Kill" # Change button to "Force Kill" IMMEDIATELY (before any async operations) if not is_force_kill: logger.info("🛑 Stop button pressed - changing to Force Kill") - self.execution_state = "force_kill_ready" + self.execution_state = ManagerExecutionState.FORCE_KILL_READY + # Clear stale server info so state can properly reset when plates are terminal + self.set_execution_server_info(None) self.update_button_states() QApplication.processEvents() + else: + # Force-kill requested: immediately disable stop interactions while + # cancellation propagates from background threads. + self.execution_state = ManagerExecutionState.STOPPING + # Clear stale server info so state can properly reset when plates are terminal + self.set_execution_server_info(None) + self.update_button_states() + + self._batch_workflow_service.stop_execution(force=is_force_kill) - self._zmq_service.stop_execution(force=is_force_kill) - def action_code_plate(self): """Generate Python code for selected plates and their pipelines (Tier 3).""" logger.debug("Code button pressed - generating Python code for plates") @@ -870,10 +1187,14 @@ def action_code_plate(self): selected_items = self.get_selected_items() if not selected_items: if self.plates: - logger.info("Code button pressed with no selection, falling back to all plates.") + logger.info( + "Code button pressed with no selection, falling back to all plates." + ) selected_items = list(self.plates) else: - logger.info("Code button pressed with no plates configured; generating empty template.") + logger.info( + "Code button pressed with no plates configured; generating empty template." + ) selected_items = [] try: @@ -883,13 +1204,15 @@ def action_code_plate(self): per_plate_configs = {} # Store pipeline config for each plate for plate_data in selected_items: - plate_path = plate_data['path'] + plate_path = plate_data["path"] plate_paths.append(plate_path) # Get pipeline definition for this plate definition_pipeline = self._get_current_pipeline_definition(plate_path) if not definition_pipeline: - logger.warning(f"No pipeline defined for {plate_data['name']}, using empty pipeline") + logger.warning( + f"No pipeline defined for {plate_data['name']}, using empty pipeline" + ) definition_pipeline = [] pipeline_data[plate_path] = definition_pipeline @@ -924,17 +1247,24 @@ def action_code_plate(self): ) editor_service = SimpleCodeEditorService(self) - use_external = os.environ.get('OPENHCS_USE_EXTERNAL_EDITOR', '').lower() in ('1', 'true', 'yes') + use_external = os.environ.get( + "OPENHCS_USE_EXTERNAL_EDITOR", "" + ).lower() in ("1", "true", "yes") code_data = { - 'clean_mode': True, 'plate_paths': plate_paths, - 'pipeline_data': pipeline_data, 'global_config': self.global_config, - 'per_plate_configs': per_plate_configs, - 'pipeline_config': pipeline_config, + "clean_mode": True, + "plate_paths": plate_paths, + "pipeline_data": pipeline_data, + "global_config": self.global_config, + "per_plate_configs": per_plate_configs, + "pipeline_config": pipeline_config, } editor_service.edit_code( - initial_content=python_code, title="Edit Orchestrator Configuration", - callback=self._handle_edited_code, use_external=use_external, - code_type='orchestrator', code_data=code_data + initial_content=python_code, + title="Edit Orchestrator Configuration", + callback=self._handle_edited_code, + use_external=use_external, + code_type="orchestrator", + code_data=code_data, ) except Exception as e: @@ -973,8 +1303,9 @@ def _ensure_plate_entries_from_code(self, plate_paths: List[str]) -> None: logger.info(f"Added plate '{plate_name}' from orchestrator code") if added_count: - # Update root ObjectState - root_state.update_parameter("orchestrator_scope_ids", new_paths) + # Atomic: register orchestrator(s) + update orchestrator_scope_ids together + with ObjectStateRegistry.atomic("register orchestrators"): + root_state.update_parameter("orchestrator_scope_ids", new_paths) if self.item_list: self.update_item_list() @@ -991,27 +1322,29 @@ def _get_orchestrator_for_path(self, plate_path: str): def _pre_code_execution(self) -> None: """Open pipeline editor window before processing orchestrator code.""" main_window = self._find_main_window() - if main_window and hasattr(main_window, 'show_pipeline_editor'): + if main_window and hasattr(main_window, "show_pipeline_editor"): main_window.show_pipeline_editor() def _apply_executed_code(self, namespace: dict) -> bool: """Extract orchestrator variables from namespace and apply to widget state.""" - if 'plate_paths' not in namespace or 'pipeline_data' not in namespace: + if "plate_paths" not in namespace or "pipeline_data" not in namespace: return False - new_plate_paths = namespace['plate_paths'] - new_pipeline_data = namespace['pipeline_data'] + new_plate_paths = namespace["plate_paths"] + new_pipeline_data = namespace["pipeline_data"] self._ensure_plate_entries_from_code(new_plate_paths) # Update global config if present - if 'global_config' in namespace: - self._apply_global_config_from_code(namespace['global_config']) + if "global_config" in namespace: + self._apply_global_config_from_code(namespace["global_config"]) # Handle per-plate configs (preferred) or single pipeline_config (legacy) - if 'per_plate_configs' in namespace: - self._apply_per_plate_configs_from_code(namespace['per_plate_configs']) - elif 'pipeline_config' in namespace: - self._apply_legacy_pipeline_config_from_code(namespace['pipeline_config'], new_plate_paths) + if "per_plate_configs" in namespace: + self._apply_per_plate_configs_from_code(namespace["per_plate_configs"]) + elif "pipeline_config" in namespace: + self._apply_legacy_pipeline_config_from_code( + namespace["pipeline_config"], new_plate_paths + ) # Update pipeline data for ALL affected plates self._apply_pipeline_data_from_code(new_pipeline_data) @@ -1029,7 +1362,7 @@ def _apply_global_config_from_code(self, new_global_config) -> None: # Apply to all orchestrators for plate in self.plates: - orchestrator = ObjectStateRegistry.get_object(plate['path']) + orchestrator = ObjectStateRegistry.get_object(plate["path"]) if orchestrator: self._update_orchestrator_global_config(orchestrator, new_global_config) @@ -1038,7 +1371,7 @@ def _apply_global_config_from_code(self, new_global_config) -> None: self.global_config_changed.emit() # Broadcast to event bus - self._broadcast_to_event_bus('config', new_global_config) + self._broadcast_to_event_bus("config", new_global_config) def _apply_per_plate_configs_from_code(self, per_plate_configs: dict) -> None: """Apply per-plate pipeline configs from executed code.""" @@ -1051,21 +1384,29 @@ def _apply_per_plate_configs_from_code(self, per_plate_configs: dict) -> None: if orchestrator: orchestrator.apply_pipeline_config(new_pipeline_config) effective_config = orchestrator.get_effective_config() - self.orchestrator_config_changed.emit(str(orchestrator.plate_path), effective_config) - logger.debug(f"Applied per-plate pipeline config to orchestrator: {orchestrator.plate_path}") + self.orchestrator_config_changed.emit( + str(orchestrator.plate_path), effective_config + ) + logger.debug( + f"Applied per-plate pipeline config to orchestrator: {orchestrator.plate_path}" + ) else: - logger.info(f"Stored pipeline config for {plate_key}; will apply when initialized.") + logger.info( + f"Stored pipeline config for {plate_key}; will apply when initialized." + ) last_pipeline_config = new_pipeline_config # Broadcast last config to event bus if last_pipeline_config: - self._broadcast_to_event_bus('config', last_pipeline_config) + self._broadcast_to_event_bus("config", last_pipeline_config) - def _apply_legacy_pipeline_config_from_code(self, new_pipeline_config, plate_paths: list) -> None: + def _apply_legacy_pipeline_config_from_code( + self, new_pipeline_config, plate_paths: list + ) -> None: """Apply legacy single pipeline_config to all plates.""" # Broadcast to event bus - self._broadcast_to_event_bus('config', new_pipeline_config) + self._broadcast_to_event_bus("config", new_pipeline_config) # Apply to all affected orchestrators for plate_path in plate_paths: @@ -1074,21 +1415,27 @@ def _apply_legacy_pipeline_config_from_code(self, new_pipeline_config, plate_pat orchestrator.apply_pipeline_config(new_pipeline_config) effective_config = orchestrator.get_effective_config() self.orchestrator_config_changed.emit(str(plate_path), effective_config) - logger.debug(f"Applied tier 3 pipeline config to orchestrator: {plate_path}") + logger.debug( + f"Applied tier 3 pipeline config to orchestrator: {plate_path}" + ) def _apply_pipeline_data_from_code(self, new_pipeline_data: dict) -> None: """Apply pipeline data for ALL affected plates with proper state invalidation.""" - if not self.pipeline_editor or not hasattr(self.pipeline_editor, '_update_pipeline_steps'): + if not self.pipeline_editor or not hasattr( + self.pipeline_editor, "_update_pipeline_steps" + ): logger.warning("No pipeline editor available to update pipeline data") self.pipeline_data_changed.emit() return - current_plate = getattr(self.pipeline_editor, 'current_plate', None) + current_plate = getattr(self.pipeline_editor, "current_plate", None) for plate_path, new_steps in new_pipeline_data.items(): # Update pipeline data via ObjectState (not dict assignment - plate_pipelines is read-only) self.pipeline_editor._update_pipeline_steps(plate_path, new_steps) - logger.debug(f"Updated pipeline for {plate_path} with {len(new_steps)} steps") + logger.debug( + f"Updated pipeline for {plate_path} with {len(new_steps)} steps" + ) # Invalidate orchestrator state self._invalidate_orchestrator_compilation_state(plate_path) @@ -1098,8 +1445,10 @@ def _apply_pipeline_data_from_code(self, new_pipeline_data: dict) -> None: self.pipeline_editor.pipeline_steps = new_steps self.pipeline_editor.update_item_list() self.pipeline_editor.pipeline_changed.emit(new_steps) - self._broadcast_to_event_bus('pipeline', new_steps) - logger.debug(f"Triggered UI cascade refresh for current plate: {plate_path}") + self._broadcast_to_event_bus("pipeline", new_steps) + logger.debug( + f"Triggered UI cascade refresh for current plate: {plate_path}" + ) self.pipeline_data_changed.emit() @@ -1133,25 +1482,27 @@ def action_view_metadata(self): return for item in selected_items: - plate_path = item['path'] + plate_path = item["path"] # Check if orchestrator is initialized orchestrator = ObjectStateRegistry.get_object(plate_path) if not orchestrator: - self.service_adapter.show_error_dialog(f"Plate must be initialized to view: {plate_path}") + self.service_adapter.show_error_dialog( + f"Plate must be initialized to view: {plate_path}" + ) continue try: # Create plate viewer window with tabs (Image Browser + Metadata) - viewer = PlateViewerWindow( - orchestrator=orchestrator, - color_scheme=self.color_scheme, - parent=self - ) + viewer = PlateViewerWindow(orchestrator=orchestrator, parent=self) viewer.show() # Use show() instead of exec() to allow multiple windows except Exception as e: - logger.error(f"Failed to open plate viewer for {plate_path}: {e}", exc_info=True) - self.service_adapter.show_error_dialog(f"Failed to open plate viewer: {str(e)}") + logger.error( + f"Failed to open plate viewer for {plate_path}: {e}", exc_info=True + ) + self.service_adapter.show_error_dialog( + f"Failed to open plate viewer: {str(e)}" + ) # ========== UI Helper Methods ========== @@ -1172,12 +1523,15 @@ def update_button_states(self): """Update button enabled/disabled states based on selection.""" selected_plates = self.get_selected_items() has_selection = len(selected_plates) > 0 + def _plate_is_initialized(plate_dict): - orchestrator = ObjectStateRegistry.get_object(plate_dict['path']) + orchestrator = ObjectStateRegistry.get_object(plate_dict["path"]) return orchestrator and orchestrator.state != OrchestratorState.CREATED has_initialized = any(_plate_is_initialized(plate) for plate in selected_plates) - has_compiled = any(plate['path'] in self.plate_compiled_data for plate in selected_plates) + has_compiled = any( + plate["path"] in self.plate_compiled_data for plate in selected_plates + ) is_running = self.is_any_plate_running() # Update button states (logic extracted from Textual version) @@ -1190,11 +1544,11 @@ def _plate_is_initialized(plate_dict): self.buttons["view_metadata"].setEnabled(has_initialized and not is_running) # Run button - enabled if plates are compiled or if currently running (for stop) - if self.execution_state == "stopping": + if self.execution_state == ManagerExecutionState.STOPPING: # Stopping state - keep button as "Stop" but disable it self.buttons["run_plate"].setEnabled(False) self.buttons["run_plate"].setText("Stop") - elif self.execution_state == "force_kill_ready": + elif self.execution_state == ManagerExecutionState.FORCE_KILL_READY: # Force kill ready state - button is "Force Kill" and enabled self.buttons["run_plate"].setEnabled(True) self.buttons["run_plate"].setText("Force Kill") @@ -1206,7 +1560,25 @@ def _plate_is_initialized(plate_dict): # Idle state - button is "Run" and enabled if plates are compiled self.buttons["run_plate"].setEnabled(has_compiled) self.buttons["run_plate"].setText("Run") - + + def refresh_execution_ui(self) -> None: + """Refresh list row statuses and action buttons after execution state changes.""" + self.update_item_list() + self.update_button_states() + + def clear_plate_execution_tracking( + self, plate_path: str, *, clear_terminal: bool = True + ) -> None: + """Clear per-plate execution runtime tracking. + + By default this also clears terminal execution status; pass ``clear_terminal=False`` + to preserve a terminal outcome label until the next explicit operation. + """ + execution_id = self.plate_execution_ids.pop(plate_path, None) + self.execution_runtime.clear_plate(plate_path, clear_terminal=clear_terminal) + if execution_id: + self._progress_tracker.clear_execution(execution_id) + def is_any_plate_running(self) -> bool: """ Check if any plate is currently running. @@ -1214,24 +1586,23 @@ def is_any_plate_running(self) -> bool: Returns: True if any plate is running, False otherwise """ - # Consider "running", "stopping", and "force_kill_ready" states as "busy" - return self.execution_state in ("running", "stopping", "force_kill_ready") - + return self.execution_state in BUSY_MANAGER_STATES + # Event handlers (on_selection_changed, on_plates_reordered, on_item_double_clicked) # provided by AbstractManagerWidget base class # Plate-specific behavior implemented via abstract hooks below - + def on_orchestrator_state_changed(self, plate_path: str, state: str): """ Handle orchestrator state changes. - + Args: plate_path: Path of the plate state: New orchestrator state """ self.update_item_list() logger.debug(f"Orchestrator state changed: {plate_path} -> {state}") - + def on_config_changed(self, new_config: GlobalPipelineConfig): """ Handle global configuration changes. @@ -1245,7 +1616,7 @@ def on_config_changed(self, new_config: GlobalPipelineConfig): # This rebuilds their pipeline configs preserving concrete values count = 0 for plate in self.plates: - orchestrator = ObjectStateRegistry.get_object(plate['path']) + orchestrator = ObjectStateRegistry.get_object(plate["path"]) if orchestrator: self._update_orchestrator_global_config(orchestrator, new_config) count += 1 @@ -1274,15 +1645,13 @@ def _get_current_pipeline_definition(self, plate_path: str) -> List: if not self.pipeline_editor: logger.warning("No pipeline editor reference - using empty pipeline") return [] - - # Get pipeline for specific plate (same logic as Textual TUI) - if hasattr(self.pipeline_editor, 'plate_pipelines') and plate_path in self.pipeline_editor.plate_pipelines: - pipeline_steps = self.pipeline_editor.plate_pipelines[plate_path] - logger.debug(f"Found pipeline for plate {plate_path} with {len(pipeline_steps)} steps") - return pipeline_steps - else: - logger.debug(f"No pipeline found for plate {plate_path}, using empty pipeline") - return [] + pipeline_steps = self.pipeline_editor.get_pipeline_for_plate(plate_path) + logger.debug( + "Loaded pipeline for plate %s from ObjectState with %d steps", + plate_path, + len(pipeline_steps), + ) + return pipeline_steps def set_pipeline_editor(self, pipeline_editor): """ @@ -1334,7 +1703,7 @@ def _validate_delete(self, items: List[Any]) -> bool: def _perform_delete(self, items: List[Any]) -> None: """Remove plates from backing list and cleanup orchestrators (required abstract method).""" - paths_to_delete = {plate['path'] for plate in items} + paths_to_delete = {plate["path"] for plate in items} # Remove from root ObjectState root_state = self._ensure_root_state() @@ -1349,7 +1718,9 @@ def _perform_delete(self, items: List[Any]) -> None: # Cascade unregister: plate + all steps + all functions (prevents memory leak) # This also removes the orchestrator from ObjectState (single source of truth) count = ObjectStateRegistry.unregister_scope_and_descendants(path_str) - logger.debug(f"Cascade unregistered {count} ObjectState(s) for deleted plate: {path}") + logger.debug( + f"Cascade unregistered {count} ObjectState(s) for deleted plate: {path}" + ) # Delete saved PipelineConfig (prevents resurrection) if path_str in self.plate_configs: @@ -1376,7 +1747,7 @@ def _format_item_content(self, item: Any, index: int, context: Any) -> str: def _get_list_item_tooltip(self, item: Any) -> str: """Get plate tooltip with orchestrator status.""" - orchestrator = ObjectStateRegistry.get_object(item['path']) + orchestrator = ObjectStateRegistry.get_object(item["path"]) if orchestrator: return f"Status: {orchestrator.state.value}" return "" @@ -1392,7 +1763,7 @@ def _get_scope_for_item(self, item: Any) -> str: """PlateManager: scope = plate_path (from orchestrator or dict).""" if isinstance(item, PipelineOrchestrator): return str(item.plate_path) - return item.get('path', '') if isinstance(item, dict) else '' + return item.get("path", "") if isinstance(item, dict) else "" # === CrossWindowPreviewMixin Hook === @@ -1404,11 +1775,15 @@ def _get_current_orchestrator(self): def _handle_compilation_error(self, plate_name: str, error_message: str): """Handle compilation error on main thread (slot).""" - self.service_adapter.show_error_dialog(f"Compilation failed for {plate_name}: {error_message}") + self.service_adapter.show_error_dialog( + f"Compilation failed for {plate_name}: {error_message}" + ) def _handle_initialization_error(self, plate_name: str, error_message: str): """Handle initialization error on main thread (slot).""" - self.service_adapter.show_error_dialog(f"Failed to initialize {plate_name}: {error_message}") + self.service_adapter.show_error_dialog( + f"Failed to initialize {plate_name}: {error_message}" + ) def _handle_execution_error(self, error_message: str): """Handle execution error on main thread (slot).""" diff --git a/openhcs/pyqt_gui/widgets/shared/plate_view_widget.py b/openhcs/pyqt_gui/widgets/shared/plate_view_widget.py index c529db67c..e5556bbd4 100644 --- a/openhcs/pyqt_gui/widgets/shared/plate_view_widget.py +++ b/openhcs/pyqt_gui/widgets/shared/plate_view_widget.py @@ -8,16 +8,108 @@ import logging from typing import Set, List, Optional, Tuple from PyQt6.QtWidgets import ( - QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QPushButton, - QLabel, QFrame, QButtonGroup + QWidget, + QVBoxLayout, + QHBoxLayout, + QGridLayout, + QPushButton, + QLabel, + QFrame, + QButtonGroup, + QSizePolicy, ) -from PyQt6.QtCore import Qt, pyqtSignal +from PyQt6.QtCore import Qt, pyqtSignal, QSize from pyqt_reactive.theming import ColorScheme from pyqt_reactive.theming import StyleSheetGenerator logger = logging.getLogger(__name__) +class SquareButton(QPushButton): + """QPushButton that fills its grid cell.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Use Expanding policy so buttons fill available space uniformly + self.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) + + +class AspectRatioContainer(QWidget): + """Container that maintains aspect ratio for its child widget. + + This widget acts as a wrapper that sizes its child to maintain a specific + aspect ratio while centering it within the available space. + """ + + # Minimum cell size in pixels (wells won't shrink smaller than this) + MIN_CELL_SIZE = 8 + + def __init__(self, child_widget: QWidget, *args, **kwargs): + super().__init__(*args, **kwargs) + self.aspect_ratio = 1.0 # width / height ratio + self.num_cols = 1 + self.num_rows = 1 + self.child_widget = child_widget + self.child_widget.setParent(self) + # Use Expanding to fill available space + self.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) + + def set_aspect_ratio(self, num_cols: int, num_rows: int): + """Set the aspect ratio based on grid dimensions.""" + self.num_cols = num_cols + self.num_rows = num_rows + if num_rows > 0: + self.aspect_ratio = num_cols / num_rows + # Set minimum size on container so layout respects it + min_width = num_cols * self.MIN_CELL_SIZE + min_height = num_rows * self.MIN_CELL_SIZE + self.setMinimumSize(min_width, min_height) + self._update_child_geometry() + + def resizeEvent(self, event): + """Resize child to maintain aspect ratio, centered in available space.""" + super().resizeEvent(event) + self._update_child_geometry() + + def _update_child_geometry(self): + """Calculate and set child geometry to maintain aspect ratio.""" + if self.aspect_ratio <= 0: + return + + available_w = self.width() + available_h = self.height() + + if available_w <= 0 or available_h <= 0: + return + + # Calculate minimum size based on cell count + min_width = self.num_cols * self.MIN_CELL_SIZE + min_height = self.num_rows * self.MIN_CELL_SIZE + + # Calculate the largest size that fits while maintaining aspect ratio + height_for_width = int(available_w / self.aspect_ratio) + width_for_height = int(available_h * self.aspect_ratio) + + if height_for_width <= available_h: + # Width-constrained: use full width + child_w = available_w + child_h = height_for_width + else: + # Height-constrained: use full height + child_w = width_for_height + child_h = available_h + + # Enforce minimum size (cells won't shrink below MIN_CELL_SIZE) + child_w = max(child_w, min_width) + child_h = max(child_h, min_height) + + # Center the child widget (may overflow if below minimum) + x = (available_w - child_w) // 2 + y = (available_h - child_h) // 2 + + self.child_widget.setGeometry(x, y, child_w, child_h) + + class PlateViewWidget(QWidget): """ Visual plate grid widget with clickable wells. @@ -37,18 +129,20 @@ class PlateViewWidget(QWidget): wells_selected = pyqtSignal(set) detach_requested = pyqtSignal() - + def __init__(self, color_scheme: Optional[ColorScheme] = None, parent=None): super().__init__(parent) - + self.color_scheme = color_scheme or ColorScheme() self.style_gen = StyleSheetGenerator(self.color_scheme) - + # State self.well_buttons = {} # well_id -> QPushButton self.wells_with_images = set() # Set of well IDs that have images self.selected_wells = set() # Currently selected wells self.plate_dimensions = (8, 12) # rows, cols (default 96-well) + self.row_offset = 0 # Offset for tight bounding box (first row index - 1) + self.col_offset = 0 # Offset for tight bounding box (first col index - 1) self.subdirs = [] # List of subdirectory names self.active_subdir = None # Currently selected subdirectory self.coord_to_well = {} # (row_index, col_index) -> well_id mapping @@ -61,28 +155,39 @@ def __init__(self, color_scheme: Optional[ColorScheme] = None, parent=None): self.drag_selection_mode = None # 'select' or 'deselect' self.drag_affected_wells = set() # Wells affected by current drag operation self.pre_drag_selection = set() # Selection state before drag started + self.drag_moved = False # Track if mouse actually moved during drag + + # Rectangle selection state (for dragging in empty space) + self.is_rect_selecting = False + self.rect_start_pos = None + self.rect_current_pos = None + self.selection_rect_widget = None # Visual rectangle overlay # Column filter sync - self.well_filter_widget = None # Reference to ColumnFilterWidget for 'well' column + self.well_filter_widget = ( + None # Reference to ColumnFilterWidget for 'well' column + ) # UI components self.subdir_buttons = {} # subdir_name -> QPushButton self.subdir_button_group = None self.well_grid_layout = None self.status_label = None - + self._setup_ui() - + def _setup_ui(self): """Setup the UI layout.""" layout = QVBoxLayout(self) layout.setContentsMargins(5, 5, 5, 5) layout.setSpacing(5) - + # Header with title, detach button, and clear button header_layout = QHBoxLayout() title_label = QLabel("Plate View") - title_label.setStyleSheet(f"font-weight: bold; color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};") + title_label.setStyleSheet( + f"font-weight: bold; color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};" + ) header_layout.addWidget(title_label) header_layout.addStretch() @@ -96,79 +201,91 @@ def _setup_ui(self): clear_btn = QPushButton("Clear Selection") clear_btn.setStyleSheet(self.style_gen.generate_button_style()) - clear_btn.clicked.connect(self.clear_selection) + # Use lambda to avoid clicked signal's bool arg being passed to clear_selection + clear_btn.clicked.connect(lambda: self.clear_selection()) header_layout.addWidget(clear_btn) layout.addLayout(header_layout) - + # Subdirectory selector (initially hidden) self.subdir_frame = QFrame() self.subdir_layout = QHBoxLayout(self.subdir_frame) self.subdir_layout.setContentsMargins(0, 0, 0, 0) self.subdir_layout.setSpacing(5) - + subdir_label = QLabel("Plate Output:") self.subdir_layout.addWidget(subdir_label) - + self.subdir_button_group = QButtonGroup(self) self.subdir_button_group.setExclusive(True) - + self.subdir_layout.addStretch() self.subdir_frame.setVisible(False) layout.addWidget(self.subdir_frame) - - # Well grid container + + # Well grid container (background color shows through as grid lines) grid_container = QFrame() - grid_container.setStyleSheet(f"background-color: {self.color_scheme.to_hex(self.color_scheme.panel_bg)}; border-radius: 3px;") + # Use panel_bg color for grid lines (shows between 1px spacing) + grid_container.setStyleSheet( + f"background-color: {self.color_scheme.to_hex(self.color_scheme.panel_bg)}; border-radius: 3px;" + ) grid_layout_wrapper = QVBoxLayout(grid_container) grid_layout_wrapper.setContentsMargins(10, 10, 10, 10) - # Create a centered widget for the grid - grid_center_widget = QWidget() - grid_center_layout = QHBoxLayout(grid_center_widget) - grid_center_layout.setContentsMargins(0, 0, 0, 0) - - # Add stretches to center the grid - grid_center_layout.addStretch() - - # Grid widget with mouse tracking for drag selection - grid_widget = QWidget() - grid_widget.setMouseTracking(True) - self.well_grid_layout = QGridLayout(grid_widget) - self.well_grid_layout.setSpacing(3) # Slightly more spacing + # Inner grid widget that holds the actual grid layout + inner_grid_widget = QWidget() + inner_grid_widget.setMouseTracking(True) + self.well_grid_layout = QGridLayout(inner_grid_widget) + self.well_grid_layout.setSpacing(2) # Thin grid lines (2px for visibility) self.well_grid_layout.setContentsMargins(0, 0, 0, 0) - self.grid_widget = grid_widget # Store reference - - grid_center_layout.addWidget(grid_widget) - grid_center_layout.addStretch() - grid_layout_wrapper.addWidget(grid_center_widget) + # Wrap in AspectRatioContainer to maintain square cells + aspect_container = AspectRatioContainer(inner_grid_widget) + self.grid_widget = inner_grid_widget # Store reference to inner widget + self.aspect_container = aspect_container # Store reference to container + + # Install event filter on inner grid widget for rectangle selection + inner_grid_widget.installEventFilter(self) + + # Create selection rectangle overlay (initially hidden) + self.selection_rect_widget = QLabel(inner_grid_widget) + self.selection_rect_widget.setStyleSheet(f""" + background-color: {self.color_scheme.to_hex(self.color_scheme.selection_bg)}40; + border: 2px solid {self.color_scheme.to_hex(self.color_scheme.selection_bg)}; + """) + self.selection_rect_widget.hide() + self.selection_rect_widget.setAttribute( + Qt.WidgetAttribute.WA_TransparentForMouseEvents + ) + self.selection_rect_widget.raise_() # Ensure it's on top + + # Add aspect container to wrapper (it will expand and center the grid) + grid_layout_wrapper.addWidget(aspect_container, 1) # stretch factor 1 to expand layout.addWidget(grid_container, 1) # Stretch to fill - + # Status label self.status_label = QLabel("No wells") - self.status_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)};") + self.status_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)};" + ) layout.addWidget(self.status_label) - # Install event filter on grid widget for drag selection - self.grid_widget.installEventFilter(self) - def set_subdirectories(self, subdirs: List[str]): """ Set available subdirectories for plate outputs. - + Args: subdirs: List of subdirectory names """ self.subdirs = subdirs - + # Clear existing buttons for btn in self.subdir_buttons.values(): self.subdir_button_group.removeButton(btn) btn.deleteLater() self.subdir_buttons.clear() - + if len(subdirs) == 0: # No subdirs, hide selector self.subdir_frame.setVisible(False) @@ -180,31 +297,39 @@ def set_subdirectories(self, subdirs: List[str]): else: # Multiple subdirs, show selector self.subdir_frame.setVisible(True) - + # Create button for each subdir for subdir in subdirs: btn = QPushButton(subdir) btn.setCheckable(True) btn.setStyleSheet(self.style_gen.generate_button_style()) - btn.clicked.connect(lambda checked, s=subdir: self._on_subdir_selected(s)) - + btn.clicked.connect( + lambda checked, s=subdir: self._on_subdir_selected(s) + ) + self.subdir_button_group.addButton(btn) - self.subdir_layout.insertWidget(self.subdir_layout.count() - 1, btn) # Before stretch + self.subdir_layout.insertWidget( + self.subdir_layout.count() - 1, btn + ) # Before stretch self.subdir_buttons[subdir] = btn - + # Auto-select first subdir if subdirs: first_btn = self.subdir_buttons[subdirs[0]] first_btn.setChecked(True) self.active_subdir = subdirs[0] - + def _on_subdir_selected(self, subdir: str): """Handle subdirectory selection.""" self.active_subdir = subdir # Could emit signal here if needed for filtering by subdir - - def set_available_wells(self, well_ids: Set[str], plate_dimensions: Optional[Tuple[int, int]] = None, - coord_to_well: Optional[dict] = None): + + def set_available_wells( + self, + well_ids: Set[str], + plate_dimensions: Optional[Tuple[int, int]] = None, + coord_to_well: Optional[dict] = None, + ): """ Update which wells have images and rebuild grid. @@ -215,17 +340,38 @@ def set_available_wells(self, well_ids: Set[str], plate_dimensions: Optional[Tup Required for non-standard well ID formats (e.g., Opera Phenix R01C01). """ self.wells_with_images = well_ids - self.coord_to_well = coord_to_well or {} + + # If coord_to_well not provided, build it from well_ids (for standard formats) + # IMPORTANT: Do this BEFORE calling _detect_dimensions + if coord_to_well is None: + self.coord_to_well = {} + for well_id in well_ids: + # Parse standard well ID format (A01, B12, etc.) + row_part = "".join(c for c in well_id if c.isalpha()) + col_part = "".join(c for c in well_id if c.isdigit()) + + if row_part and col_part: + # Convert row letter to index (A=1, B=2, etc.) + row_idx = sum( + (ord(c.upper()) - ord("A") + 1) * (26**i) + for i, c in enumerate(reversed(row_part)) + ) + col_idx = int(col_part) + self.coord_to_well[(row_idx, col_idx)] = well_id + else: + self.coord_to_well = coord_to_well # Build reverse mapping (well_id -> coord) - self.well_to_coord = {well_id: coord for coord, well_id in self.coord_to_well.items()} + self.well_to_coord = { + well_id: coord for coord, well_id in self.coord_to_well.items() + } if not well_ids: self._clear_grid() self.status_label.setText("No wells") return - # Use provided dimensions or auto-detect + # Use provided dimensions or auto-detect (NOW coord_to_well is populated) if plate_dimensions is not None: self.plate_dimensions = plate_dimensions else: @@ -236,118 +382,244 @@ def set_available_wells(self, well_ids: Set[str], plate_dimensions: Optional[Tup # Update status self._update_status() - + def _detect_dimensions(self, well_ids: Set[str]) -> Tuple[int, int]: """ - Auto-detect plate dimensions from well IDs. - - Assumes well IDs are already parsed (e.g., 'A01', 'B03'). - Extracts max row letter and max column number. - + Auto-detect plate dimensions from well IDs (tight bounding box). + + Only includes rows/columns that contain wells with images. + Args: well_ids: Set of well IDs - + Returns: (rows, cols) tuple """ + if not well_ids: + return (8, 12) # Default + + min_row = float("inf") max_row = 0 + min_col = float("inf") max_col = 0 - - for well_id in well_ids: - # Extract row letter(s) and column number(s) - row_part = ''.join(c for c in well_id if c.isalpha()) - col_part = ''.join(c for c in well_id if c.isdigit()) - - if row_part: - # Convert row letter to index (A=1, B=2, AA=27, etc.) - row_idx = sum((ord(c.upper()) - ord('A') + 1) * (26 ** i) - for i, c in enumerate(reversed(row_part))) - max_row = max(max_row, row_idx) - - if col_part: - max_col = max(max_col, int(col_part)) - - return (max_row, max_col) - + + # If we have coord_to_well mapping, use that directly + if self.coord_to_well: + for (row_idx, col_idx), well_id in self.coord_to_well.items(): + if well_id in well_ids: + min_row = min(min_row, row_idx) + max_row = max(max_row, row_idx) + min_col = min(min_col, col_idx) + max_col = max(max_col, col_idx) + else: + # Parse well IDs to extract coordinates + for well_id in well_ids: + # Extract row letter(s) and column number(s) + row_part = "".join(c for c in well_id if c.isalpha()) + col_part = "".join(c for c in well_id if c.isdigit()) + + if row_part: + # Convert row letter to index (A=1, B=2, AA=27, etc.) + row_idx = sum( + (ord(c.upper()) - ord("A") + 1) * (26**i) + for i, c in enumerate(reversed(row_part)) + ) + min_row = min(min_row, row_idx) + max_row = max(max_row, row_idx) + + if col_part: + col_idx = int(col_part) + min_col = min(min_col, col_idx) + max_col = max(max_col, col_idx) + + # Store offset for coordinate mapping + self.row_offset = min_row - 1 if min_row != float("inf") else 0 + self.col_offset = min_col - 1 if min_col != float("inf") else 0 + + rows = max_row - min_row + 1 if max_row > 0 else 8 + cols = max_col - min_col + 1 if max_col > 0 else 12 + + return (rows, cols) + def _clear_grid(self): """Clear the well grid.""" for btn in self.well_buttons.values(): btn.deleteLater() self.well_buttons.clear() - + # Clear layout while self.well_grid_layout.count(): item = self.well_grid_layout.takeAt(0) if item.widget(): item.widget().deleteLater() - + def _build_grid(self): """Build the well grid based on current dimensions.""" self._clear_grid() - rows, cols = self.plate_dimensions + # Get unique rows and columns that actually have wells + actual_rows = sorted(set(coord[0] for coord in self.coord_to_well.keys())) + actual_cols = sorted(set(coord[1] for coord in self.coord_to_well.keys())) + + if not actual_rows or not actual_cols: + return - # Add column headers (1, 2, 3, ...) - for col in range(1, cols + 1): - header = QLabel(str(col)) - header.setAlignment(Qt.AlignmentFlag.AlignCenter) - header.setFixedSize(40, 20) # Fixed size for consistent spacing - header.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)}; font-size: 11px;") - self.well_grid_layout.addWidget(header, 0, col) + # Get bounding rectangle (min to max, inclusive) + min_row, max_row = min(actual_rows), max(actual_rows) + min_col, max_col = min(actual_cols), max(actual_cols) + + # Create full range including empty positions + all_rows = list(range(min_row, max_row + 1)) + all_cols = list(range(min_col, max_col + 1)) + + # Calculate minimum size based on label width + # Find longest column number for width calculation + max_col_label = str(max_col) + from PyQt6.QtGui import QFontMetrics + from PyQt6.QtGui import QFont + + font = QFont() + font.setPointSize(10) + fm = QFontMetrics(font) + min_col_width = max( + fm.horizontalAdvance(max_col_label) + 8, 15 + ) # +8 for padding, min 15px + min_row_height = max(fm.height() + 4, 15) # +4 for padding, min 15px + + # Header minimum sizes (can be smaller than well size) + min_header_width = max(min_col_width, 8) + min_header_height = max(min_row_height, 8) + + # Well buttons use MIN_CELL_SIZE directly - allow shrinking independently of headers + min_well_size = AspectRatioContainer.MIN_CELL_SIZE + + # Top-left corner: Invert selection button + invert_btn = QPushButton("⇄") + invert_btn.setFlat(True) + invert_btn.setCursor(Qt.CursorShape.PointingHandCursor) + invert_btn.setMinimumSize(18, 18) + invert_btn.setToolTip("Invert Selection") + invert_btn.setStyleSheet(f""" + QPushButton {{ + color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)}; + font-size: 12px; + border: none; + background: transparent; + }} + QPushButton:hover {{ + color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; + background-color: {self.color_scheme.to_hex(self.color_scheme.panel_bg)}; + }} + """) + invert_btn.clicked.connect(lambda: self._invert_selection()) + self.well_grid_layout.addWidget(invert_btn, 0, 0) + + # Add column headers - for all columns in bounding rectangle + for grid_col, actual_col in enumerate(all_cols, start=1): + header = QPushButton(str(actual_col)) + header.setFlat(True) + header.setCursor(Qt.CursorShape.PointingHandCursor) + header.setMinimumSize(min_header_width, 18) + header.setMaximumHeight(18) + header.setSizePolicy(QSizePolicy.Policy.Ignored, QSizePolicy.Policy.Fixed) + header.setStyleSheet(f""" + QPushButton {{ + color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)}; + font-size: 10px; + border: none; + background: transparent; + }} + QPushButton:hover {{ + color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; + background-color: {self.color_scheme.to_hex(self.color_scheme.panel_bg)}; + }} + """) + header.clicked.connect( + lambda checked, c=actual_col: self._toggle_column_selection(c) + ) + self.well_grid_layout.addWidget(header, 0, grid_col) - # Add row headers and well buttons - for row in range(1, rows + 1): + # Add row headers and well buttons - for all rows in bounding rectangle + for grid_row, actual_row in enumerate(all_rows, start=1): # Row header (A, B, C, ...) - row_letter = self._index_to_row_letter(row) - header = QLabel(row_letter) - header.setAlignment(Qt.AlignmentFlag.AlignCenter) - header.setFixedSize(20, 40) # Fixed size for consistent spacing - header.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)}; font-size: 11px;") - self.well_grid_layout.addWidget(header, row, 0) - - # Well buttons - for col in range(1, cols + 1): - # Use coordinate mapping if available, otherwise construct standard well ID - well_id = self.coord_to_well.get((row, col), f"{row_letter}{col:02d}") - - btn = QPushButton() - btn.setFixedSize(40, 40) # Larger buttons for better visibility + row_letter = self._index_to_row_letter(actual_row) + header = QPushButton(row_letter) + header.setFlat(True) + header.setCursor(Qt.CursorShape.PointingHandCursor) + header.setMinimumSize(18, min_header_height) + header.setMaximumWidth(18) + header.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Ignored) + header.setStyleSheet(f""" + QPushButton {{ + color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)}; + font-size: 10px; + border: none; + background: transparent; + }} + QPushButton:hover {{ + color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; + background-color: {self.color_scheme.to_hex(self.color_scheme.panel_bg)}; + }} + """) + header.clicked.connect( + lambda checked, r=actual_row: self._toggle_row_selection(r) + ) + self.well_grid_layout.addWidget(header, grid_row, 0) + + # Well buttons - for all columns in bounding rectangle + for grid_col, actual_col in enumerate(all_cols, start=1): + # Check if this coordinate has a well + well_id = self.coord_to_well.get((actual_row, actual_col)) + + btn = SquareButton() # Use SquareButton to maintain 1:1 aspect ratio + btn.setMinimumSize(min_well_size, min_well_size) btn.setCheckable(True) - # Set initial state - if well_id in self.wells_with_images: + if well_id and well_id in self.wells_with_images: + # Well exists and has images btn.setEnabled(True) - btn.setStyleSheet(self._get_well_button_style('has_images')) + btn.setStyleSheet(self._get_well_button_style("has_images")) + btn.clicked.connect( + lambda checked, wid=well_id: self._on_well_clicked(wid, checked) + ) + btn.setProperty("well_id", well_id) + btn.installEventFilter(self) + self.well_buttons[well_id] = btn else: + # Empty position in bounding rectangle btn.setEnabled(False) - btn.setStyleSheet(self._get_well_button_style('empty')) + btn.setStyleSheet(self._get_well_button_style("empty")) + btn.setProperty("well_id", None) + # Install event filter for rectangle selection + btn.installEventFilter(self) - btn.clicked.connect(lambda checked, wid=well_id: self._on_well_clicked(wid, checked)) + self.well_grid_layout.addWidget(btn, grid_row, grid_col) - # Store well_id in button for lookup - btn.setProperty('well_id', well_id) + # Set uniform column and row stretches so all cells get equal space + # This ensures wells expand uniformly and stay aligned + for grid_col in range(1, len(all_cols) + 1): + self.well_grid_layout.setColumnStretch(grid_col, 1) + for grid_row in range(1, len(all_rows) + 1): + self.well_grid_layout.setRowStretch(grid_row, 1) - self.well_grid_layout.addWidget(btn, row, col) - self.well_buttons[well_id] = btn + # Set aspect ratio on container to maintain square wells + # Add 1 to account for header row/column + self.aspect_container.set_aspect_ratio(len(all_cols) + 1, len(all_rows) + 1) - # Update reverse mapping if not using coord_to_well - if (row, col) not in self.coord_to_well: - self.well_to_coord[well_id] = (row, col) - def _index_to_row_letter(self, index: int) -> str: """Convert row index to letter(s) (1=A, 2=B, 27=AA, etc.).""" result = "" while index > 0: index -= 1 - result = chr(ord('A') + (index % 26)) + result + result = chr(ord("A") + (index % 26)) + result index //= 26 return result - + def _get_well_button_style(self, state: str) -> str: """Generate style for well button based on state.""" cs = self.color_scheme - - if state == 'empty': + + if state == "empty": return f""" QPushButton {{ background-color: {cs.to_hex(cs.button_disabled_bg)}; @@ -356,7 +628,7 @@ def _get_well_button_style(self, state: str) -> str: border-radius: 3px; }} """ - elif state == 'selected': + elif state == "selected": return f""" QPushButton {{ background-color: {cs.to_hex(cs.selection_bg)}; @@ -377,7 +649,7 @@ def _get_well_button_style(self, state: str) -> str: background-color: {cs.to_hex(cs.button_hover_bg)}; }} """ - + def _on_well_clicked(self, well_id: str, checked: bool): """Handle well button click (only for non-drag clicks).""" # Skip if this was part of a drag operation @@ -386,15 +658,19 @@ def _on_well_clicked(self, well_id: str, checked: bool): if checked: self.selected_wells.add(well_id) - self.well_buttons[well_id].setStyleSheet(self._get_well_button_style('selected')) + self.well_buttons[well_id].setStyleSheet( + self._get_well_button_style("selected") + ) else: self.selected_wells.discard(well_id) - self.well_buttons[well_id].setStyleSheet(self._get_well_button_style('has_images')) + self.well_buttons[well_id].setStyleSheet( + self._get_well_button_style("has_images") + ) self._update_status() - self.wells_selected.emit(self.selected_wells.copy()) self.sync_to_well_filter() - + self.wells_selected.emit(self.selected_wells.copy()) + def clear_selection(self, emit_signal: bool = True, sync_to_filter: bool = True): """ Clear all selected wells. @@ -403,20 +679,31 @@ def clear_selection(self, emit_signal: bool = True, sync_to_filter: bool = True) emit_signal: Whether to emit wells_selected signal (default True) sync_to_filter: Whether to sync to well filter (default True) """ + logger.info( + f"[CLEAR] clear_selection called, had {len(self.selected_wells)} wells, emit_signal={emit_signal}, sync_to_filter={sync_to_filter}" + ) for well_id in list(self.selected_wells): if well_id in self.well_buttons: btn = self.well_buttons[well_id] btn.setChecked(False) - btn.setStyleSheet(self._get_well_button_style('has_images')) + btn.setStyleSheet(self._get_well_button_style("has_images")) self.selected_wells.clear() self._update_status() + logger.info("[CLEAR] After clear, about to sync") - if emit_signal: - self.wells_selected.emit(set()) - + # Sync to well filter BEFORE emitting signal + # This ensures the Well column filter has all values selected + # before _apply_combined_filters() runs if sync_to_filter: + logger.info("[CLEAR] Syncing to well filter") self.sync_to_well_filter() + logger.info("[CLEAR] Done syncing to well filter") + + if emit_signal: + logger.info("[CLEAR] Emitting wells_selected with empty set") + self.wells_selected.emit(set()) + logger.info("[CLEAR] Done emitting") def select_wells(self, well_ids: Set[str], emit_signal: bool = True): """ @@ -434,13 +721,13 @@ def select_wells(self, well_ids: Set[str], emit_signal: bool = True): self.selected_wells.add(well_id) btn = self.well_buttons[well_id] btn.setChecked(True) - btn.setStyleSheet(self._get_well_button_style('selected')) + btn.setStyleSheet(self._get_well_button_style("selected")) self._update_status() if emit_signal: - self.wells_selected.emit(self.selected_wells.copy()) self.sync_to_well_filter() - + self.wells_selected.emit(self.selected_wells.copy()) + def _update_status(self): """Update status label.""" total_wells = len(self.wells_with_images) @@ -454,57 +741,170 @@ def _update_status(self): self.status_label.setText(f"{total_wells} wells have images") def eventFilter(self, obj, event): - """Handle mouse events on grid widget for drag selection.""" - if obj != self.grid_widget: - return super().eventFilter(obj, event) - - from PyQt6.QtCore import QEvent - - if event.type() == QEvent.Type.MouseButtonPress: - if event.button() == Qt.MouseButton.LeftButton: - # Find which button is under the cursor - child = self.grid_widget.childAt(event.pos()) - if isinstance(child, QPushButton): - well_id = child.property('well_id') + """Handle mouse events on buttons for drag selection and rectangle selection in empty space.""" + from PyQt6.QtCore import QEvent, QRect + + # Handle events from well buttons (both active and empty) + if isinstance(obj, QPushButton): + well_id = obj.property("well_id") + + if event.type() == QEvent.Type.MouseButtonPress: + if event.button() == Qt.MouseButton.LeftButton: + # Always start rectangle selection for visual feedback + self.is_rect_selecting = True + self.rect_start_pos = obj.mapTo(self.grid_widget, event.pos()) + self.rect_current_pos = self.rect_start_pos + self.pre_drag_selection = self.selected_wells.copy() + + # Grab mouse on grid widget so we get events even outside buttons + self.grid_widget.grabMouse() + + # Show rectangle at starting position (even if small) + rect = QRect( + self.rect_start_pos, self.rect_current_pos + ).normalized() + self.selection_rect_widget.setGeometry(rect) + self.selection_rect_widget.raise_() # Bring to front + self.selection_rect_widget.show() + if well_id and well_id in self.wells_with_images: - # Start drag selection - save current selection state + # Also track drag selection on active well for immediate toggle self.is_dragging = True self.drag_start_well = well_id self.drag_current_well = well_id self.drag_affected_wells = set() - self.pre_drag_selection = self.selected_wells.copy() + self.drag_moved = False # Track if mouse actually moved # Determine selection mode - self.drag_selection_mode = 'deselect' if well_id in self.selected_wells else 'select' + self.drag_selection_mode = ( + "deselect" if well_id in self.selected_wells else "select" + ) # Apply to starting well - self._toggle_well_selection(well_id, self.drag_selection_mode == 'select') + self._toggle_well_selection( + well_id, self.drag_selection_mode == "select" + ) self.drag_affected_wells.add(well_id) - # Emit signal immediately + # Emit signal immediately for drag start self._update_status() - self.wells_selected.emit(self.selected_wells.copy()) self.sync_to_well_filter() + self.wells_selected.emit(self.selected_wells.copy()) - elif event.type() == QEvent.Type.MouseMove: - if self.is_dragging and event.buttons() & Qt.MouseButton.LeftButton: - # Find which button is under the cursor - child = self.grid_widget.childAt(event.pos()) - if isinstance(child, QPushButton): - well_id = child.property('well_id') - if well_id and well_id in self.wells_with_images: - if well_id != self.drag_current_well: - self.drag_current_well = well_id - self._update_drag_selection() - - elif event.type() == QEvent.Type.MouseButtonRelease: - if event.button() == Qt.MouseButton.LeftButton and self.is_dragging: - # End drag selection - self.is_dragging = False - self.drag_start_well = None - self.drag_current_well = None - self.drag_selection_mode = None - self.drag_affected_wells.clear() + # Accept event to prevent button's clicked signal + event.accept() + return True + + elif event.type() == QEvent.Type.MouseMove: + if ( + self.is_rect_selecting + and event.buttons() & Qt.MouseButton.LeftButton + ): + # Update rectangle from button (always update rectangle during any drag) + self.rect_current_pos = obj.mapTo(self.grid_widget, event.pos()) + rect = QRect( + self.rect_start_pos, self.rect_current_pos + ).normalized() + self.selection_rect_widget.setGeometry(rect) + self.selection_rect_widget.raise_() # Keep on top + + # Find wells inside rectangle and select them + self._update_rectangle_selection(rect) + + if self.is_dragging: + self.drag_moved = True # Mark that we actually dragged + + # Accept event to prevent default handling + event.accept() + return True + + elif event.type() == QEvent.Type.MouseButtonRelease: + if event.button() == Qt.MouseButton.LeftButton: + if self.is_rect_selecting or self.is_dragging: + # Release mouse grab + self.grid_widget.releaseMouse() + + # End both drag and rectangle selection + if self.is_dragging: + self.is_dragging = False + self.drag_start_well = None + self.drag_current_well = None + self.drag_selection_mode = None + self.drag_affected_wells.clear() + self.drag_moved = False + + if self.is_rect_selecting: + self.is_rect_selecting = False + self.rect_start_pos = None + self.rect_current_pos = None + self.selection_rect_widget.hide() + + # Accept event to prevent button's clicked signal + event.accept() + return True + + # Handle events from grid widget (empty space or grabbed mouse) for rectangle selection + elif obj == self.grid_widget: + if event.type() == QEvent.Type.MouseButtonPress: + if event.button() == Qt.MouseButton.LeftButton: + # Check if we clicked on empty space (not a button) + target = self.grid_widget.childAt(event.pos()) + if not isinstance(target, QPushButton): + # Start rectangle selection + self.is_rect_selecting = True + self.rect_start_pos = event.pos() + self.rect_current_pos = event.pos() + self.pre_drag_selection = self.selected_wells.copy() + + # Grab mouse so we get events even outside the widget + self.grid_widget.grabMouse() + + # Show rectangle at starting position + rect = QRect( + self.rect_start_pos, self.rect_current_pos + ).normalized() + self.selection_rect_widget.setGeometry(rect) + self.selection_rect_widget.raise_() # Bring to front + self.selection_rect_widget.show() + + event.accept() + return True + + elif event.type() == QEvent.Type.MouseMove: + if ( + self.is_rect_selecting + and event.buttons() & Qt.MouseButton.LeftButton + ): + # Update rectangle + self.rect_current_pos = event.pos() + rect = QRect( + self.rect_start_pos, self.rect_current_pos + ).normalized() + self.selection_rect_widget.setGeometry(rect) + self.selection_rect_widget.raise_() # Keep on top + + # Find wells inside rectangle and select them + self._update_rectangle_selection(rect) + + event.accept() + return True + + elif event.type() == QEvent.Type.MouseButtonRelease: + if ( + event.button() == Qt.MouseButton.LeftButton + and self.is_rect_selecting + ): + # Release mouse grab + self.grid_widget.releaseMouse() + + # End rectangle selection + self.is_rect_selecting = False + self.rect_start_pos = None + self.rect_current_pos = None + self.selection_rect_widget.hide() + + event.accept() + return True return super().eventFilter(obj, event) @@ -518,11 +918,11 @@ def _toggle_well_selection(self, well_id: str, select: bool): if select and well_id not in self.selected_wells: self.selected_wells.add(well_id) btn.setChecked(True) - btn.setStyleSheet(self._get_well_button_style('selected')) + btn.setStyleSheet(self._get_well_button_style("selected")) elif not select and well_id in self.selected_wells: self.selected_wells.discard(well_id) btn.setChecked(False) - btn.setStyleSheet(self._get_well_button_style('has_images')) + btn.setStyleSheet(self._get_well_button_style("has_images")) def _update_drag_selection(self): """Update selection for all wells in the drag rectangle.""" @@ -559,15 +959,128 @@ def _update_drag_selection(self): # Apply selection to all wells in current rectangle for well_id in wells_in_rectangle: - self._toggle_well_selection(well_id, self.drag_selection_mode == 'select') + self._toggle_well_selection(well_id, self.drag_selection_mode == "select") # Update affected wells to current rectangle self.drag_affected_wells = wells_in_rectangle.copy() - # Emit signal and sync to well filter + # Sync to well filter BEFORE emitting signal self._update_status() + self.sync_to_well_filter() self.wells_selected.emit(self.selected_wells.copy()) + + def _update_rectangle_selection(self, rect): + """Update selection based on rectangle drawn in empty space.""" + from PyQt6.QtCore import QRect + + # Find all wells whose buttons intersect with the rectangle + wells_in_rect = set() + for well_id, btn in self.well_buttons.items(): + if well_id not in self.wells_with_images: + continue + + # Get button geometry in grid widget coordinates + btn_rect = QRect(btn.pos(), btn.size()) + + # Check if button intersects with selection rectangle + if rect.intersects(btn_rect): + wells_in_rect.add(well_id) + + # Restore pre-drag selection for wells outside the rectangle + for well_id in self.wells_with_images: + if well_id not in wells_in_rect: + # Restore to pre-drag state + should_be_selected = well_id in self.pre_drag_selection + self._toggle_well_selection(well_id, should_be_selected) + else: + # Select wells in rectangle + self._toggle_well_selection(well_id, True) + + # Sync to well filter BEFORE emitting signal + self._update_status() self.sync_to_well_filter() + self.wells_selected.emit(self.selected_wells.copy()) + + def _toggle_row_selection(self, row_index: int): + """Toggle selection for all wells in a row.""" + # Find all wells in this row + wells_in_row = [] + for well_id, coord in self.well_to_coord.items(): + if coord[0] == row_index and well_id in self.wells_with_images: + wells_in_row.append(well_id) + + if not wells_in_row: + return + + # Check if all wells in row are selected + all_selected = all(well_id in self.selected_wells for well_id in wells_in_row) + + # If all selected, deselect all; otherwise select all + select = not all_selected + + for well_id in wells_in_row: + self._toggle_well_selection(well_id, select) + + # Sync to well filter BEFORE emitting signal + self._update_status() + self.sync_to_well_filter() + self.wells_selected.emit(self.selected_wells.copy()) + + def _toggle_column_selection(self, col_index: int): + """Toggle selection for all wells in a column.""" + # Find all wells in this column + wells_in_col = [] + for well_id, coord in self.well_to_coord.items(): + if coord[1] == col_index and well_id in self.wells_with_images: + wells_in_col.append(well_id) + + if not wells_in_col: + return + + # Check if all wells in column are selected + all_selected = all(well_id in self.selected_wells for well_id in wells_in_col) + + # If all selected, deselect all; otherwise select all + select = not all_selected + + for well_id in wells_in_col: + self._toggle_well_selection(well_id, select) + + # Sync to well filter BEFORE emitting signal + self._update_status() + self.sync_to_well_filter() + self.wells_selected.emit(self.selected_wells.copy()) + + def _invert_selection(self): + """Invert the current selection (select unselected, deselect selected).""" + # Get all wells with images + all_wells = self.wells_with_images.copy() + + logger.info( + f"[INVERT] Before: selected_wells={len(self.selected_wells)}, all_wells={len(all_wells)}" + ) + logger.info(f"[INVERT] selected_wells={self.selected_wells}") + + # Calculate inverted selection + new_selection = all_wells - self.selected_wells + logger.info(f"[INVERT] new_selection={len(new_selection)} wells") + + # Clear current selection + for well_id in self.selected_wells.copy(): + self._toggle_well_selection(well_id, False) + + # Apply new selection + for well_id in new_selection: + self._toggle_well_selection(well_id, True) + + logger.info(f"[INVERT] After: selected_wells={len(self.selected_wells)}") + logger.info(f"[INVERT] selected_wells={self.selected_wells}") + + # Sync to well filter BEFORE emitting signal + # This ensures the Well column filter is updated before _apply_combined_filters() runs + self._update_status() + self.sync_to_well_filter() + self.wells_selected.emit(self.selected_wells.copy()) def set_well_filter_widget(self, well_filter_widget): """ @@ -583,11 +1096,11 @@ def sync_to_well_filter(self): if not self.well_filter_widget: return - # Update well filter checkboxes to match plate view selection - # Block signals to prevent circular sync loop - # If no wells selected in plate view, select all in filter (show all) + # Block signals for performance - table updates via wells_selected signal if self.selected_wells: - self.well_filter_widget.set_selected_values(self.selected_wells, block_signals=True) + self.well_filter_widget.set_selected_values( + self.selected_wells, block_signals=True + ) else: self.well_filter_widget.select_all(block_signals=True) @@ -601,4 +1114,3 @@ def sync_from_well_filter(self): # Update plate view to match (without emitting signal to avoid loop) self.select_wells(selected_in_filter, emit_signal=False) - diff --git a/openhcs/pyqt_gui/widgets/shared/server_browser/__init__.py b/openhcs/pyqt_gui/widgets/shared/server_browser/__init__.py new file mode 100644 index 000000000..226ae26f6 --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/server_browser/__init__.py @@ -0,0 +1,23 @@ +"""Server browser composition helpers.""" + +from .progress_tree_builder import ProgressNode, ProgressTreeBuilder +from .presentation_models import ( + ProgressTopologyState, + ExecutionServerSummary, + summarize_execution_server, + ServerRowPresenter, +) +from .server_kill_service import ServerKillPlan, ServerKillService +from .server_tree_population import ServerTreePopulation + +__all__ = [ + "ProgressNode", + "ProgressTreeBuilder", + "ProgressTopologyState", + "ExecutionServerSummary", + "summarize_execution_server", + "ServerKillPlan", + "ServerKillService", + "ServerRowPresenter", + "ServerTreePopulation", +] diff --git a/openhcs/pyqt_gui/widgets/shared/server_browser/presentation_models.py b/openhcs/pyqt_gui/widgets/shared/server_browser/presentation_models.py new file mode 100644 index 000000000..91406af70 --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/server_browser/presentation_models.py @@ -0,0 +1,204 @@ +"""Server-browser state + presentation helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import singledispatchmethod +from typing import Callable, Dict, Iterable, List + +from PyQt6.QtWidgets import QTreeWidgetItem + +from openhcs.core.progress import ProgressChannel, ProgressEvent, phase_channel +from pyqt_reactive.services import ( + BaseServerInfo, + ExecutionServerInfo, + GenericServerInfo, + ViewerServerInfo, +) + +from .progress_tree_builder import ProgressNode + + +class ProgressTopologyState: + """Tracks and validates execution topology claims from progress events.""" + + def __init__(self) -> None: + self.known_wells: Dict[tuple[str, str], List[str]] = {} + self.worker_assignments: Dict[tuple[str, str], Dict[str, List[str]]] = {} + self.seen_execution_ids: set[str] = set() + # Track step names per well: {(exec_id, plate_id, axis_id): {step_index: step_name}} + self.step_names: Dict[tuple[str, str, str], Dict[int, str]] = {} + + def register_event(self, event: ProgressEvent) -> None: + execution_id = event.execution_id + plate_id = event.plate_id + topology_key = (execution_id, plate_id) + + self.seen_execution_ids.add(execution_id) + + if event.total_wells: + self.known_wells[topology_key] = list(event.total_wells) + if event.worker_assignments is not None: + self.worker_assignments[topology_key] = event.worker_assignments + + # Capture step names from INIT event (emitted once per execution) + if ( + event.step_name == "" + and event.phase.value == "init" + and event.axis_id == "" + ): + if event.step_names: + # step_names applies to all wells in this execution + for well_id in self.known_wells.get(topology_key, []): + well_key = (execution_id, plate_id, well_id) + self.step_names[well_key] = { + i: name for i, name in enumerate(event.step_names) + } + + event_channel = phase_channel(event.phase) + if ( + event_channel in {ProgressChannel.PIPELINE, ProgressChannel.STEP} + and event.axis_id + and topology_key not in self.worker_assignments + ): + raise ValueError( + f"Execution event arrived without worker_assignments for plate '{plate_id}'" + ) + + if event.worker_slot is not None: + if topology_key not in self.worker_assignments: + raise ValueError( + f"Worker event arrived before assignments for plate '{plate_id}'" + ) + assignments = self.worker_assignments[topology_key] + if event.worker_slot not in assignments: + raise ValueError( + f"Unknown worker slot '{event.worker_slot}' for plate '{plate_id}'" + ) + expected = sorted(assignments[event.worker_slot]) + actual = sorted(event.owned_wells or []) + if actual != expected: + raise ValueError( + f"Worker claim mismatch for slot '{event.worker_slot}': expected={expected}, got={actual}" + ) + + def clear_execution(self, execution_id: str) -> None: + self.seen_execution_ids.discard(execution_id) + keys_to_remove = [ + key for key in self.known_wells.keys() if key[0] == execution_id + ] + for key in keys_to_remove: + self.known_wells.pop(key, None) + self.worker_assignments.pop(key, None) + + def clear_all(self) -> None: + self.known_wells.clear() + self.worker_assignments.clear() + self.seen_execution_ids.clear() + + +@dataclass(frozen=True) +class ExecutionServerSummary: + """Derived status for execution-server top-level row.""" + + status_text: str + info_text: str + + +def summarize_execution_server(nodes: Iterable[ProgressNode]) -> ExecutionServerSummary: + node_list = list(nodes) + if not node_list: + return ExecutionServerSummary(status_text="✅ Idle", info_text="") + + queued = sum(1 for node in node_list if node.status == "⏳ Queued") + compiling = sum(1 for node in node_list if node.status == "⏳ Compiling") + executing = sum(1 for node in node_list if node.status == "⚙️ Executing") + compiled = sum(1 for node in node_list if node.status == "✅ Compiled") + completed = sum(1 for node in node_list if node.status == "✅ Complete") + failed = sum(1 for node in node_list if node.status.startswith("❌")) + overall_pct = sum(node.percent for node in node_list) / len(node_list) + + parts: list[str] = [] + if queued > 0: + parts.append(f"⏳ {queued} queued") + if compiling > 0: + parts.append(f"⏳ {compiling} compiling") + if executing > 0: + parts.append(f"⚙️ {executing} executing") + if compiled > 0: + parts.append(f"✅ {compiled} compiled") + if completed > 0: + parts.append(f"✅ {completed} complete") + if failed > 0: + parts.append(f"❌ {failed} failed") + + status_text = ", ".join(parts) if parts else "✅ Idle" + info_text = f"Avg: {overall_pct:.1f}% | {len(node_list)} plates" + return ExecutionServerSummary(status_text=status_text, info_text=info_text) + + +class ServerRowPresenter: + """Type-dispatched server row rendering and child population.""" + + def __init__( + self, + *, + create_tree_item: Callable[[str, str, str, dict], QTreeWidgetItem], + update_execution_server_item: Callable[[QTreeWidgetItem, dict], None], + log_warning: Callable[..., None], + ) -> None: + self._create_tree_item = create_tree_item + self._update_execution_server_item = update_execution_server_item + self._log_warning = log_warning + + @singledispatchmethod + def render_server(self, info: BaseServerInfo, status_icon: str) -> QTreeWidgetItem: + raise NotImplementedError(f"No render for {type(info).__name__}") + + @render_server.register + def _(self, info: ExecutionServerInfo, status_icon: str) -> QTreeWidgetItem: + server_text = f"Port {info.port} - Execution Server" + if not info.ready: + return self._create_tree_item(server_text, "🚀 Starting", "", info.raw) + info_text = f"{len(info.workers)} active workers" if info.workers else "" + return self._create_tree_item(server_text, "✅ Idle", info_text, info.raw) + + @render_server.register + def _(self, info: ViewerServerInfo, status_icon: str) -> QTreeWidgetItem: + kind_name = info.viewer_kind.name.title() + display_text = f"Port {info.port} - {kind_name} Viewer" + info_text = "" + if info.memory_mb is not None: + info_text = f"Mem: {info.memory_mb:.0f}MB" + if info.cpu_percent is not None: + info_text += f" | CPU: {info.cpu_percent:.1f}%" + return self._create_tree_item(display_text, status_icon, info_text, info.raw) + + @render_server.register + def _(self, info: GenericServerInfo, status_icon: str) -> QTreeWidgetItem: + return self._create_tree_item( + f"Port {info.port} - {info.server_name}", status_icon, "", info.raw + ) + + @singledispatchmethod + def populate_server_children( + self, info: BaseServerInfo, server_item: QTreeWidgetItem + ) -> bool: + self._log_warning( + "_populate_server_children: No handler for type %s, using default (no children)", + type(info).__name__, + ) + return False + + @populate_server_children.register + def _(self, info: ExecutionServerInfo, server_item: QTreeWidgetItem) -> bool: + self._update_execution_server_item(server_item, info.raw) + return server_item.childCount() > 0 + + @populate_server_children.register + def _(self, info: ViewerServerInfo, server_item: QTreeWidgetItem) -> bool: + return False + + @populate_server_children.register + def _(self, info: GenericServerInfo, server_item: QTreeWidgetItem) -> bool: + return False diff --git a/openhcs/pyqt_gui/widgets/shared/server_browser/progress_tree_builder.py b/openhcs/pyqt_gui/widgets/shared/server_browser/progress_tree_builder.py new file mode 100644 index 000000000..e2d3bf239 --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/server_browser/progress_tree_builder.py @@ -0,0 +1,499 @@ +"""Build execution/compilation progress trees for the server browser.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + +from pyqt_reactive.strategies import ( + ExplicitPercentTreeAggregationPolicy, + MeanTreeAggregationPolicy, + TreeAggregationPolicyRegistry, +) +from pyqt_reactive.widgets.shared import TreeNode + +from openhcs.core.progress import ( + ProgressChannel, + ProgressEvent, + phase_channel, + is_failure_event, + is_success_terminal_event, +) + + +@dataclass +class ProgressNode: + """Pure tree node model for progress rendering.""" + + node_id: str + node_type: str + label: str + status: str + info: str + execution_id: str | None = None + percent: float = 0.0 + aggregation_policy_id: str = "mean" + children: List["ProgressNode"] = field(default_factory=list) + + +_NODE_AGGREGATION_POLICY_BY_TYPE: Dict[str, str] = { + "plate": "mean", + "worker": "mean", + "well": "mean", + "step": "explicit", + "compilation": "explicit", +} + +_TREE_AGGREGATION_REGISTRY = TreeAggregationPolicyRegistry( + policies={ + "mean": MeanTreeAggregationPolicy(), + "explicit": ExplicitPercentTreeAggregationPolicy(), + } +) + + +class ProgressTreeBuilder: + """Transforms ProgressEvent snapshots into hierarchical progress nodes.""" + + def build_progress_tree( + self, + *, + executions: Dict[str, List[ProgressEvent]], + worker_assignments: Dict[tuple[str, str], Dict[str, List[str]]], + known_wells: Dict[tuple[str, str], List[str]], + step_names: Dict[tuple[str, str, str], Dict[int, str]], + get_plate_name: Callable[[str, str | None], str], + ) -> List[ProgressNode]: + plates: Dict[tuple[str, str], Dict[str, List[ProgressEvent]]] = {} + for exec_id, events_list in executions.items(): + for event in events_list: + key = (exec_id, event.plate_id) + if key not in plates: + plates[key] = {"events": []} + plates[key]["events"].append(event) + + nodes_by_plate: Dict[str, tuple[float, ProgressNode]] = {} + for (exec_id, plate_id), pdata in plates.items(): + events = pdata["events"] + if not events: + continue + latest_timestamp = max((event.timestamp for event in events), default=0.0) + plate_name = get_plate_name(plate_id, exec_id) + is_executing = self._is_execution_mode( + execution_id=exec_id, + plate_id=plate_id, + events=events, + worker_assignments=worker_assignments, + ) + if is_executing: + children = self._build_worker_children( + execution_id=exec_id, + plate_id=plate_id, + events=events, + worker_assignments=worker_assignments, + step_names=step_names, + ) + else: + children = self._build_compilation_children( + execution_id=exec_id, + plate_id=plate_id, + events=events, + known_wells=known_wells, + ) + + plate_node = ProgressNode( + node_id=plate_id, + node_type="plate", + label=f"📋 {plate_name}", + status="⚙️ Executing" if is_executing else "⏳ Compiling", + info="", + execution_id=exec_id, + children=children, + ) + self._aggregate_percent_recursive(plate_node) + if is_executing: + if self._has_failed_descendant(plate_node): + plate_node.status = "❌ Failed" + elif plate_node.percent >= 100.0: + plate_node.status = "✅ Complete" + elif self._all_leaves_queued(plate_node): + plate_node.status = "⏳ Queued" + else: + plate_node.status = "⚙️ Executing" + else: + if self._has_failed_descendant(plate_node): + plate_node.status = "❌ Compile Failed" + else: + plate_node.status = ( + "✅ Compiled" if plate_node.percent >= 100.0 else "⏳ Compiling" + ) + self._apply_node_percent_text(plate_node) + existing = nodes_by_plate.get(plate_id) + if existing is None or latest_timestamp > existing[0]: + nodes_by_plate[plate_id] = (latest_timestamp, plate_node) + + # Promote compile-only plates to "✅ Complete" when their execution- + # mode sibling was already cleaned up from the tracker. The topology + # state still holds worker_assignments for the removed execution, so + # we can detect that an execution run existed. Clear the stale + # compilation children so the tree doesn't regress to "Compiling". + # Only promote when the compile itself is finished ("✅ Compiled") — + # a fresh compile still in progress must not be overridden. + for plate_id, (_ts, node) in nodes_by_plate.items(): + if node.status == "✅ Compiled": + had_execution_sibling = any( + exec_id not in executions + for (exec_id, p_id) in worker_assignments + if p_id == plate_id + ) + if had_execution_sibling: + node.status = "✅ Complete" + node.children = [] + + return sorted( + (pair[1] for pair in nodes_by_plate.values()), key=lambda node: node.node_id + ) + + def _build_worker_children( + self, + *, + execution_id: str, + plate_id: str, + events: List[ProgressEvent], + worker_assignments: Dict[tuple[str, str], Dict[str, List[str]]], + step_names: Dict[tuple[str, str, str], Dict[int, str]], + ) -> List[ProgressNode]: + assignments = worker_assignments.get((execution_id, plate_id)) + if assignments is None: + raise ValueError( + f"Missing worker assignments for execution plate '{plate_id}'" + ) + + channels = self._partition_events_by_channel(events) + pipeline_by_axis: Dict[str, ProgressEvent] = { + event.axis_id: event + for event in channels[ProgressChannel.PIPELINE.value] + if event.axis_id + } + step_by_axis: Dict[str, ProgressEvent] = { + event.axis_id: event + for event in channels[ProgressChannel.STEP.value] + if event.axis_id + } + + worker_nodes: List[ProgressNode] = [] + execution_started = any( + phase_channel(event.phase) == ProgressChannel.INIT for event in events + ) + for worker_slot, axis_ids in sorted(assignments.items()): + well_nodes = [ + self._build_well_node( + axis_id=axis_id, + pipeline_event=pipeline_by_axis.get(axis_id), + step_event=step_by_axis.get(axis_id), + step_names=step_names.get((execution_id, plate_id, axis_id), {}), + ) + for axis_id in axis_ids + ] + failed_count = sum(1 for node in well_nodes if node.status == "❌ Failed") + complete_count = sum( + 1 for node in well_nodes if node.status == "✅ Complete" + ) + queued_count = sum(1 for node in well_nodes if node.status == "⏳ Queued") + active_count = ( + len(well_nodes) - failed_count - complete_count - queued_count + ) + if failed_count > 0: + worker_status = f"❌ {failed_count} failed" + elif complete_count == len(well_nodes): + worker_status = "✅ Complete" + elif queued_count == len(well_nodes): + worker_status = "⚙️ Starting" if execution_started else "⏳ Queued" + else: + worker_status = f"⚙️ {active_count} active" + worker_nodes.append( + ProgressNode( + node_id=worker_slot, + node_type="worker", + label=f"Worker {worker_slot}", + status=worker_status, + info="", + children=well_nodes, + ) + ) + return worker_nodes + + def _build_compilation_children( + self, + *, + execution_id: str, + plate_id: str, + events: List[ProgressEvent], + known_wells: Dict[tuple[str, str], List[str]], + ) -> List[ProgressNode]: + channels = self._partition_events_by_channel(events) + compile_by_axis: Dict[str, ProgressEvent] = { + event.axis_id: event + for event in channels[ProgressChannel.COMPILE.value] + if event.axis_id + } + known_axis_ids = list(known_wells.get((execution_id, plate_id), [])) + axis_ids = known_axis_ids if known_axis_ids else sorted(compile_by_axis.keys()) + extra_axis_ids = [ + axis_id for axis_id in compile_by_axis if axis_id not in axis_ids + ] + axis_ids.extend(sorted(extra_axis_ids)) + + compilation_nodes: List[ProgressNode] = [] + for axis_id in axis_ids: + compile_event = compile_by_axis.get(axis_id) + if compile_event is None: + status = "⏳ Compiling" + percent = 0.0 + elif self._is_failure_event(compile_event): + status = "❌ Failed" + percent = compile_event.percent + elif self._is_success_terminal_event(compile_event): + status = "✅ Compiled" + percent = compile_event.percent + else: + status = "⏳ Compiling" + percent = compile_event.percent + + compilation_nodes.append( + ProgressNode( + node_id=axis_id, + node_type="compilation", + label=f"[{axis_id}]", + status=status, + info="", + percent=percent, + aggregation_policy_id="explicit", + ) + ) + return compilation_nodes + + @staticmethod + def _partition_events_by_channel( + events: List[ProgressEvent], + ) -> Dict[str, List[ProgressEvent]]: + partitioned: Dict[str, List[ProgressEvent]] = { + ProgressChannel.INIT.value: [], + ProgressChannel.COMPILE.value: [], + ProgressChannel.PIPELINE.value: [], + ProgressChannel.STEP.value: [], + } + for event in events: + partitioned[phase_channel(event.phase).value].append(event) + return partitioned + + def _build_well_node( + self, + *, + axis_id: str, + pipeline_event: Optional[ProgressEvent], + step_event: Optional[ProgressEvent], + step_names: Dict[int, str], + ) -> ProgressNode: + if pipeline_event is None: + status = "⏳ Queued" + percent = 0.0 + elif ProgressTreeBuilder._is_failure_event(pipeline_event): + status = "❌ Failed" + percent = pipeline_event.percent + elif ProgressTreeBuilder._is_success_terminal_event(pipeline_event): + status = "✅ Complete" + percent = pipeline_event.percent + else: + status = f"⚙️ {pipeline_event.step_name}" + percent = pipeline_event.percent + + # Build step nodes for ALL steps (completed + current + future) + # This ensures the mean aggregation calculates overall progress correctly + children: List[ProgressNode] = [] + if pipeline_event is not None and pipeline_event.total > 0: + current_step_idx = pipeline_event.completed + total_steps = pipeline_event.total + + # Add completed steps at 100% + for step_idx in range(current_step_idx): + step_name = step_names.get(step_idx, f"Step {step_idx + 1}") + children.append( + ProgressNode( + node_id=f"{axis_id}_step_{step_idx}", + node_type="step", + label=f"🔧 {step_idx + 1} - {step_name}", + status="✅ Complete", + info="100.0%", + percent=100.0, + aggregation_policy_id="explicit", + ) + ) + + # Add current step with actual progress + if current_step_idx < total_steps: + step_name = step_names.get( + current_step_idx, f"Step {current_step_idx + 1}" + ) + if ( + step_event is not None + and step_event.step_name == pipeline_event.step_name + ): + if ProgressTreeBuilder._is_failure_event(step_event): + step_status = "❌ Failed" + step_percent = step_event.percent + else: + step_status = ( + f"{step_event.completed}/{step_event.total} groups" + ) + step_percent = step_event.percent + else: + # Step event not yet available for current step + step_status = "⏳ Starting" + step_percent = 0.0 + + children.append( + ProgressNode( + node_id=f"{axis_id}_step_{current_step_idx}", + node_type="step", + label=f"🔧 {current_step_idx + 1} - {step_name}", + status=step_status, + info=f"{step_percent:.1f}%", + percent=step_percent, + aggregation_policy_id="explicit", + ) + ) + + # Add future steps at 0% to ensure proper average calculation + for step_idx in range(current_step_idx + 1, total_steps): + step_name = step_names.get(step_idx, f"Step {step_idx + 1}") + children.append( + ProgressNode( + node_id=f"{axis_id}_step_{step_idx}", + node_type="step", + label=f"🔧 {step_idx + 1} - {step_name}", + status="⏳ Pending", + info="0.0%", + percent=0.0, + aggregation_policy_id="explicit", + ) + ) + + return ProgressNode( + node_id=axis_id, + node_type="well", + label=f"[{axis_id}]", + status=status, + info="", + percent=percent, + aggregation_policy_id="mean", + children=children, + ) + + def _aggregate_percent_recursive(self, node: ProgressNode) -> float: + if not node.children: + return node.percent + child_values = [ + self._aggregate_percent_recursive(child) for child in node.children + ] + expected_policy = _NODE_AGGREGATION_POLICY_BY_TYPE.get(node.node_type) + if expected_policy is None: + raise ValueError(f"No aggregation policy for node_type '{node.node_type}'") + if node.aggregation_policy_id != expected_policy: + raise ValueError( + f"Aggregation policy mismatch for node_type '{node.node_type}': " + f"expected '{expected_policy}', got '{node.aggregation_policy_id}'" + ) + # When node has children, aggregate only children (ignore explicit percent) + # Explicit percent is only used when there are no children + node.percent = _TREE_AGGREGATION_REGISTRY.aggregate( + node.aggregation_policy_id, 0.0, child_values + ) + return node.percent + + def _apply_node_percent_text(self, node: ProgressNode) -> None: + if node.node_type in {"plate", "worker", "well", "compilation"}: + node.info = f"{node.percent:.1f}%" + elif node.node_type == "step" and not node.info: + node.info = f"{node.percent:.1f}%" + for child in node.children: + self._apply_node_percent_text(child) + + @staticmethod + def _is_failure_event(event: ProgressEvent) -> bool: + return is_failure_event(event) + + @staticmethod + def _is_success_terminal_event(event: ProgressEvent) -> bool: + return is_success_terminal_event(event) + + @staticmethod + def _is_execution_mode( + *, + execution_id: str, + plate_id: str, + events: List[ProgressEvent], + worker_assignments: Dict[tuple[str, str], Dict[str, List[str]]], + ) -> bool: + # Phase channels are authoritative for mode selection: + # compile-channel presence means compilation view, unless we also + # have real axis-scoped pipeline/step execution events. + has_compile_events = any( + phase_channel(event.phase) == ProgressChannel.COMPILE for event in events + ) + has_axis_execution_events = any( + phase_channel(event.phase) + in {ProgressChannel.PIPELINE, ProgressChannel.STEP} + and bool(event.axis_id) + for event in events + ) + if has_compile_events and not has_axis_execution_events: + return False + + topology_key = (execution_id, plate_id) + if topology_key in worker_assignments: + return True + return has_axis_execution_events + + @staticmethod + def _all_leaves_queued(node: ProgressNode) -> bool: + """Return True only when *every* leaf descendant has '⏳ Queued' status.""" + if not node.children: + return node.status == "⏳ Queued" + return all( + ProgressTreeBuilder._all_leaves_queued(child) for child in node.children + ) + + @staticmethod + def _has_status_descendant(node: ProgressNode, status: str) -> bool: + if node.status == status: + return True + return any( + ProgressTreeBuilder._has_status_descendant(child, status) + for child in node.children + ) + + @staticmethod + def _has_failed_descendant(node: ProgressNode) -> bool: + if node.status.startswith("❌"): + return True + return any( + ProgressTreeBuilder._has_failed_descendant(child) for child in node.children + ) + + @staticmethod + def to_tree_node(node: ProgressNode) -> TreeNode: + return TreeNode( + node_id=node.node_id, + node_type=node.node_type, + label=node.label, + status=node.status, + info=node.info, + children=[ + ProgressTreeBuilder.to_tree_node(child) for child in node.children + ], + ) + + @staticmethod + def to_tree_nodes(nodes: List[ProgressNode]) -> List[TreeNode]: + return [ProgressTreeBuilder.to_tree_node(node) for node in nodes] diff --git a/openhcs/pyqt_gui/widgets/shared/server_browser/server_kill_service.py b/openhcs/pyqt_gui/widgets/shared/server_browser/server_kill_service.py new file mode 100644 index 000000000..fa05654de --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/server_browser/server_kill_service.py @@ -0,0 +1,96 @@ +"""Server kill execution service for ZMQ server browser.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable, List, Tuple + + +@dataclass(frozen=True) +class ServerKillPlan: + """Execution policy for killing selected server ports.""" + + graceful: bool + strict_failures: bool + emit_signal_on_failure: bool + success_message: str + + +class ServerKillService: + """Performs server kill operations with explicit policy.""" + + def __init__( + self, + kill_server_fn: Callable[[int, bool, object], bool] | None = None, + queue_tracker_registry_factory: Callable[[], object] | None = None, + config: object | None = None, + ) -> None: + if kill_server_fn is None: + from zmqruntime.client import ZMQClient + + kill_server_fn = ( + lambda port, graceful, cfg: ZMQClient.kill_server_on_port( + port, graceful=graceful, config=cfg + ) + ) + if queue_tracker_registry_factory is None: + from zmqruntime.queue_tracker import GlobalQueueTrackerRegistry + + queue_tracker_registry_factory = GlobalQueueTrackerRegistry + if config is None: + from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG + + config = OPENHCS_ZMQ_CONFIG + + self._kill_server_fn = kill_server_fn + self._queue_tracker_registry_factory = queue_tracker_registry_factory + self._config = config + + def kill_ports( + self, + *, + ports: List[int], + plan: ServerKillPlan, + on_server_killed: Callable[[int], None], + log_info: Callable[..., None], + log_warning: Callable[..., None], + log_error: Callable[..., None], + ) -> Tuple[bool, str]: + failed_ports: List[int] = [] + registry = self._queue_tracker_registry_factory() + + for port in ports: + try: + log_info("Killing server on port %s (graceful=%s)", port, plan.graceful) + success = self._kill_server_fn(port, plan.graceful, self._config) + if success: + registry.remove_tracker(port) + on_server_killed(port) + continue + + log_warning( + "kill_server_on_port returned False for port %s (graceful=%s)", + port, + plan.graceful, + ) + if plan.strict_failures: + failed_ports.append(port) + if plan.emit_signal_on_failure: + registry.remove_tracker(port) + on_server_killed(port) + except Exception as error: + log_error( + "Error killing server on port %s (graceful=%s): %s", + port, + plan.graceful, + error, + ) + if plan.strict_failures: + failed_ports.append(port) + if plan.emit_signal_on_failure: + registry.remove_tracker(port) + on_server_killed(port) + + if plan.strict_failures and failed_ports: + return False, f"Failed to quit servers on ports: {failed_ports}" + return True, plan.success_message diff --git a/openhcs/pyqt_gui/widgets/shared/server_browser/server_tree_population.py b/openhcs/pyqt_gui/widgets/shared/server_browser/server_tree_population.py new file mode 100644 index 000000000..2ff8f923f --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/server_browser/server_tree_population.py @@ -0,0 +1,148 @@ +"""Populate ZMQ server browser tree from typed server snapshots.""" + +from __future__ import annotations + +from typing import Callable, Dict, List, Set + +from PyQt6.QtWidgets import QTreeWidget, QTreeWidgetItem +from zmqruntime.viewer_state import ViewerStateManager, ViewerState + +from pyqt_reactive.services import ( + BaseServerInfo, +) + + +class ServerTreePopulation: + """Builds top-level server rows and launching/busy viewer pseudo-rows.""" + + def __init__( + self, + *, + create_tree_item: Callable[[str, str, str, dict], QTreeWidgetItem], + render_server: Callable[[BaseServerInfo, str], QTreeWidgetItem], + populate_server_children: Callable[[BaseServerInfo, QTreeWidgetItem], bool], + log_info: Callable[..., None], + ) -> None: + self._create_tree_item = create_tree_item + self._render_server = render_server + self._populate_server_children = populate_server_children + self._log_info = log_info + + def populate_tree( + self, + *, + tree: QTreeWidget, + parsed_servers: List[BaseServerInfo], + ) -> None: + from zmqruntime.queue_tracker import GlobalQueueTrackerRegistry + + launching_viewers = self._get_launching_viewers() + self._add_launching_viewers_to_tree(tree, launching_viewers) + + scanned_ports = {server_info.port for server_info in parsed_servers} + registry = GlobalQueueTrackerRegistry() + self._add_busy_viewers_to_tree( + tree=tree, + registry=registry, + scanned_ports=scanned_ports, + launching_viewer_ports=set(launching_viewers.keys()), + ) + self._add_scanned_servers_to_tree( + tree=tree, + parsed_servers=parsed_servers, + launching_viewer_ports=set(launching_viewers.keys()), + ) + + @staticmethod + def _get_launching_viewers() -> Dict[int, Dict]: + mgr = ViewerStateManager.get_instance() + return { + viewer.port: { + "type": viewer.viewer_type, + "queued_images": viewer.queued_images, + } + for viewer in mgr.list_viewers() + if viewer.state == ViewerState.LAUNCHING + } + + def _add_launching_viewers_to_tree( + self, tree: QTreeWidget, launching_viewers: Dict[int, Dict] + ) -> None: + for port, info in launching_viewers.items(): + viewer_type = info["type"].capitalize() + queued_images = info["queued_images"] + info_text = ( + f"{queued_images} images queued" if queued_images > 0 else "Starting..." + ) + item = self._create_tree_item( + f"Port {port} - {viewer_type} Viewer", + "🚀 Launching", + info_text, + {"port": port, "launching": True}, + ) + tree.addTopLevelItem(item) + + def _add_busy_viewers_to_tree( + self, + *, + tree: QTreeWidget, + registry, + scanned_ports: Set[int], + launching_viewer_ports: Set[int], + ) -> None: + for tracker_port, tracker in registry.get_all_trackers().items(): + if tracker_port in scanned_ports or tracker_port in launching_viewer_ports: + continue + pending = tracker.get_pending_count() + if pending <= 0: + continue + + processed, total = tracker.get_progress() + viewer_type = tracker.viewer_type.capitalize() + status_text = "⚙️" + info_text = f"Processing: {processed}/{total} images" + if tracker.has_stuck_images(): + status_text = "⚠️" + stuck_images = tracker.get_stuck_images() + info_text += f" (⚠️ {len(stuck_images)} stuck)" + + pseudo_server = { + "port": tracker_port, + "server": f"{viewer_type}ViewerServer", + "ready": True, + "busy": True, + } + item = self._create_tree_item( + f"Port {tracker_port} - {viewer_type}ViewerServer", + status_text, + info_text, + pseudo_server, + ) + tree.addTopLevelItem(item) + + def _add_scanned_servers_to_tree( + self, + *, + tree: QTreeWidget, + parsed_servers: List[BaseServerInfo], + launching_viewer_ports: Set[int], + ) -> None: + for server_info in parsed_servers: + if server_info.port in launching_viewer_ports: + continue + status_icon = "✅" if server_info.ready else "🚀" + server_item = self._render_server(server_info, status_icon) + if server_item is None: + continue + tree.addTopLevelItem(server_item) + self._log_info( + "About to populate children for server type %s on port %s", + type(server_info).__name__, + server_info.port, + ) + has_children = self._populate_server_children(server_info, server_item) + self._log_info( + "Populate children returned %s for port %s", + has_children, + server_info.port, + ) diff --git a/openhcs/pyqt_gui/widgets/shared/services/batch_workflow_service.py b/openhcs/pyqt_gui/widgets/shared/services/batch_workflow_service.py new file mode 100644 index 000000000..381cb245f --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/services/batch_workflow_service.py @@ -0,0 +1,974 @@ +"""Unified batch workflow service for compile + execute flows.""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +import threading +from dataclasses import dataclass +from typing import Any, Dict, List, Callable, Optional, TypeVar + +from PyQt6.QtCore import QEventLoop +from PyQt6.QtWidgets import QApplication + +from openhcs.core.orchestrator.orchestrator import OrchestratorState +from openhcs.core.progress import ProgressEvent +from openhcs.core.progress.projection import ( + ExecutionRuntimeProjection, + build_execution_runtime_projection_from_registry, +) +from openhcs.pyqt_gui.widgets.shared.services.plate_config_resolver import ( + resolve_pipeline_config_for_plate, +) +from openhcs.pyqt_gui.widgets.shared.services.progress_batch_reset import ( + reset_progress_views_for_new_batch, +) +from openhcs.pyqt_gui.widgets.shared.services.execution_server_status_presenter import ( + ExecutionServerStatusPresenter, +) +from openhcs.pyqt_gui.widgets.shared.services.execution_state import ( + STOP_PENDING_MANAGER_STATES, + ManagerExecutionState, + TerminalExecutionStatus, + parse_terminal_status, +) +from openhcs.pyqt_gui.widgets.shared.server_browser import ( + ServerKillPlan, + ServerKillService, +) +from openhcs.pyqt_gui.widgets.shared.services.zmq_client_service import ZMQClientService +from pyqt_reactive.services import ( + CallbackIntervalSnapshotPollerPolicy, + DefaultServerInfoParser, + ExecutionServerInfo, + IntervalSnapshotPoller, + ServerInfoParserABC, +) +from zmqruntime.execution import ( + BatchSubmitWaitEngine, + CallbackBatchSubmitWaitPolicy, + ExecutionStatusPoller, + CallbackExecutionStatusPollPolicy, +) + +logger = logging.getLogger(__name__) +T = TypeVar("T") + + +@dataclass(frozen=True) +class CompileJob: + """Single compile unit for a plate.""" + + plate_path: str + plate_name: str + definition_pipeline: List + pipeline_config: Any + + +class BatchWorkflowService: + """Single owner of batch compilation and execution workflow.""" + + def __init__( + self, + host, + port: int = 7777, + client_service: ZMQClientService | None = None, + server_info_parser: ServerInfoParserABC | None = None, + ) -> None: + self.host = host + self.port = port + self.client_service = client_service or ZMQClientService(port=port) + server_info_parser_impl = ( + server_info_parser + if server_info_parser is not None + else DefaultServerInfoParser() + ) + + self._progress_dirty = False + from PyQt6.QtCore import QTimer + from openhcs.pyqt_gui.config import ProgressUIConfig + + self._progress_coalesce_timer = QTimer() + self._progress_coalesce_timer.timeout.connect(self._coalesced_progress_update) + self._progress_coalesce_timer.start(ProgressUIConfig().update_interval_ms) + self._runtime_projection = ExecutionRuntimeProjection() + self._server_info_parser = server_info_parser_impl + self._server_info_poller = IntervalSnapshotPoller[ExecutionServerInfo]( + CallbackIntervalSnapshotPollerPolicy( + fetch_snapshot_fn=self._fetch_server_info_snapshot, + clone_snapshot_fn=lambda snapshot: snapshot, + poll_interval_seconds_value=1.0, + on_snapshot_changed_fn=lambda _snapshot: self._mark_progress_dirty(), + on_poll_error_fn=lambda error: logger.debug( + "Server info poll failed: %s", error + ), + ) + ) + self._compile_batch_engine = BatchSubmitWaitEngine[CompileJob]() + self._execution_status_poller = ExecutionStatusPoller() + self._server_kill_service = ServerKillService() + self._server_status_presenter = ExecutionServerStatusPresenter() + self._registry_listener = self._on_registry_event + self.host._progress_tracker.add_listener(self._registry_listener) + self._registry_listener_registered = True + self._cleaned_up = False + + def cleanup(self) -> None: + """Release timers/listeners owned by this service.""" + if self._cleaned_up: + return + self._cleaned_up = True + + if self._registry_listener_registered: + removed = self.host._progress_tracker.remove_listener( + self._registry_listener + ) + if not removed: + raise RuntimeError( + "BatchWorkflowService listener removal failed: listener not registered" + ) + self._registry_listener_registered = False + + if self._progress_coalesce_timer is not None: + self._progress_coalesce_timer.stop() + self._progress_coalesce_timer.deleteLater() + self._progress_coalesce_timer = None + + async def compile_plates(self, selected_items: List[Dict]) -> None: + """Compile pipelines for selected plates.""" + self._flush_pending_ui_edits() + reset_progress_views_for_new_batch(self.host) + self.host.emit_progress_started(len(selected_items)) + loop = asyncio.get_event_loop() + + try: + zmq_client = await self.client_service.connect( + progress_callback=self._on_progress, + persistent=True, + timeout=15, + ) + plate_paths = [str(item["path"]) for item in selected_items] + for plate_path in plate_paths: + self.host.clear_plate_execution_tracking(plate_path) + self.host.plate_compile_pending.update(plate_paths) + self.host.update_item_list() + self.host.emit_status( + f"Queueing compilation for {len(selected_items)} plate(s)..." + ) + + completed_count = 0 + compile_jobs: List[CompileJob] = [] + for plate_data in selected_items: + plate_path = str(plate_data["path"]) + try: + compile_jobs.append( + self._build_compile_job_from_plate_data(plate_data) + ) + except Exception as error: + self._handle_compile_failure(plate_data, plate_path, error) + completed_count += 1 + self.host.emit_progress_updated(completed_count) + + waiting_announced = False + + def _on_wait_success( + job: CompileJob, _execution_id: str, _idx: int, _total: int + ) -> None: + self.host.plate_compiled_data[job.plate_path] = { + "definition_pipeline": job.definition_pipeline, + } + self.host.clear_plate_execution_tracking(job.plate_path) + self._set_orchestrator_state(job.plate_path, OrchestratorState.COMPILED) + self.host.emit_orchestrator_state(job.plate_path, "COMPILED") + logger.info("Successfully compiled %s", job.plate_path) + + def _on_wait_error( + job: CompileJob, error: Exception, _idx: int, _total: int + ) -> None: + self._handle_compile_failure( + {"name": job.plate_name}, job.plate_path, error + ) + + def _on_wait_start(_job: CompileJob, _idx: int, total: int) -> None: + nonlocal waiting_announced + if waiting_announced: + return + waiting_announced = True + self.host.emit_status( + f"Queued {total} compilation job(s). Waiting for completion..." + ) + + def _on_wait_finally(job: CompileJob, _idx: int, _total: int) -> None: + nonlocal completed_count + self.host.plate_compile_pending.discard(job.plate_path) + self.host.update_item_list() + completed_count += 1 + self.host.emit_progress_updated(completed_count) + + compile_policy = self._make_compile_policy( + zmq_client=zmq_client, + loop=loop, + fail_fast_submit=False, + fail_fast_wait=False, + on_submit_error=lambda job, + error, + _idx, + _total: self._handle_compile_failure( + {"name": job.plate_name}, job.plate_path, error + ), + on_wait_start=_on_wait_start, + on_wait_success=_on_wait_success, + on_wait_error=_on_wait_error, + on_wait_finally=_on_wait_finally, + ) + await self._compile_batch_engine.run(compile_jobs, compile_policy) + finally: + if self.host.execution_state != ManagerExecutionState.RUNNING: + await self.client_service.disconnect() + + self.host.emit_progress_finished() + self.host.emit_status( + f"Compilation completed for {len(selected_items)} plate(s)" + ) + self.host.update_button_states() + + async def run_plates(self, ready_items: List[Dict]) -> None: + """Run selected plates using compile-all then execute-all workflow.""" + self._flush_pending_ui_edits() + loop = asyncio.get_event_loop() + try: + plate_paths = [str(item["path"]) for item in ready_items] + logger.info("Starting ZMQ execution for %d plates", len(plate_paths)) + + self._reset_progress_for_new_batch() + self.host.emit_clear_logs() + + await self.client_service.connect( + progress_callback=self._on_progress, persistent=True, timeout=15 + ) + + self.host.plate_execution_ids.clear() + self.host.execution_runtime.begin_batch(plate_paths) + self.host.plate_progress.clear() + + from objectstate import ObjectStateRegistry + + for item in ready_items: + plate_path = str(item["path"]) + orchestrator = ObjectStateRegistry.get_object(plate_path) + if orchestrator is not None: + orchestrator._state = OrchestratorState.EXECUTING + self.host.emit_orchestrator_state( + plate_path, OrchestratorState.EXECUTING.value + ) + + self.host.execution_state = ManagerExecutionState.RUNNING + self.host.emit_status( + f"Compiling {len(ready_items)} plate(s) before execution..." + ) + self.host.update_button_states() + self.host.update_item_list() + + run_specs = [self._build_run_spec(plate_path) for plate_path in plate_paths] + compile_artifacts = await self._compile_plates_before_execution( + run_specs=run_specs, + loop=loop, + ) + + self.host.emit_status( + f"Compilation complete. Submitting {len(run_specs)} plate(s) for execution..." + ) + for run_spec in run_specs: + plate_path = run_spec["plate_path"] + await self._submit_plate( + run_spec=run_spec, + compile_artifact_id=compile_artifacts[plate_path], + loop=loop, + ) + except Exception as error: + logger.error("Failed to execute plates via ZMQ: %s", error, exc_info=True) + self.host.emit_error(f"Failed to execute: {error}") + await self._handle_execution_failure(loop) + + async def _compile_plates_before_execution( + self, run_specs: List[Dict[str, Any]], loop + ) -> Dict[str, str]: + """Compile all selected plates before submitting execution jobs.""" + if self.client_service.zmq_client is None: + raise RuntimeError("ZMQ client is not connected") + + zmq_client = self.client_service.zmq_client + compile_jobs = [ + self._build_compile_job_from_run_spec(run_spec) for run_spec in run_specs + ] + waiting_announced = False + + def _on_wait_start(job: CompileJob, _idx: int, _total: int) -> None: + nonlocal waiting_announced + if not waiting_announced: + waiting_announced = True + self.host.emit_status( + f"Queued {len(compile_jobs)} compile job(s) before execution. Waiting for completion..." + ) + self.host.update_item_list() + + def _on_wait_success( + job: CompileJob, _execution_id: str, index: int, total: int + ) -> None: + self.host.emit_status(f"Compiled {index}/{total}: {job.plate_path}") + self.host.update_item_list() + + def _on_wait_error( + job: CompileJob, error: Exception, _idx: int, _total: int + ) -> None: + self._mark_execution_compile_failed(job.plate_path, error) + + compile_policy = self._make_compile_policy( + zmq_client=zmq_client, + loop=loop, + fail_fast_submit=True, + fail_fast_wait=True, + on_submit_error=lambda job, + error, + _idx, + _total: self._mark_execution_compile_failed(job.plate_path, error), + on_wait_start=_on_wait_start, + on_wait_success=_on_wait_success, + on_wait_error=_on_wait_error, + ) + compile_artifacts = await self._compile_batch_engine.run( + compile_jobs, compile_policy + ) + return compile_artifacts + + def _build_compile_job_from_plate_data( + self, plate_data: Dict[str, Any] + ) -> CompileJob: + plate_path = str(plate_data["path"]) + definition_pipeline = self.host.get_pipeline_definition(plate_path) + if not definition_pipeline: + logger.warning( + "No pipeline defined for %s, using empty pipeline", + plate_data["name"], + ) + definition_pipeline = [] + + self._validate_pipeline_steps(definition_pipeline) + pipeline_config = resolve_pipeline_config_for_plate(self.host, plate_path) + logger.info( + "Compile snapshot: plate=%s steps=%d fingerprint=%s step_names=%s", + plate_path, + len(definition_pipeline), + self._pipeline_fingerprint(definition_pipeline), + self._pipeline_step_names(definition_pipeline), + ) + return CompileJob( + plate_path=plate_path, + plate_name=str(plate_data["name"]), + definition_pipeline=definition_pipeline, + pipeline_config=pipeline_config, + ) + + @staticmethod + def _build_compile_job_from_run_spec(run_spec: Dict[str, Any]) -> CompileJob: + plate_path = str(run_spec["plate_path"]) + definition_pipeline = run_spec["definition_pipeline"] + logger.info( + "Compile-before-run snapshot: plate=%s steps=%d fingerprint=%s step_names=%s", + plate_path, + len(definition_pipeline), + BatchWorkflowService._pipeline_fingerprint(definition_pipeline), + BatchWorkflowService._pipeline_step_names(definition_pipeline), + ) + return CompileJob( + plate_path=plate_path, + plate_name=plate_path, + definition_pipeline=definition_pipeline, + pipeline_config=run_spec["pipeline_config"], + ) + + def _make_compile_policy( + self, + *, + zmq_client, + loop, + fail_fast_submit: bool, + fail_fast_wait: bool, + on_submit_error: Callable[[CompileJob, Exception, int, int], None] + | None = None, + on_wait_start: Callable[[CompileJob, int, int], None] | None = None, + on_wait_success: Callable[[CompileJob, str, int, int], None] | None = None, + on_wait_error: Callable[[CompileJob, Exception, int, int], None] | None = None, + on_wait_finally: Callable[[CompileJob, int, int], None] | None = None, + ) -> CallbackBatchSubmitWaitPolicy[CompileJob]: + return CallbackBatchSubmitWaitPolicy( + submit_fn=lambda job: self._submit_compile_job( + job=job, + zmq_client=zmq_client, + loop=loop, + ), + wait_fn=lambda submission_id, job: self._wait_compile_job( + submission_id=submission_id, + job=job, + zmq_client=zmq_client, + loop=loop, + ), + job_key_fn=lambda job: job.plate_path, + fail_fast_submit_value=fail_fast_submit, + fail_fast_wait_value=fail_fast_wait, + on_submit_error_fn=on_submit_error, + on_wait_start_fn=on_wait_start, + on_wait_success_fn=on_wait_success, + on_wait_error_fn=on_wait_error, + on_wait_finally_fn=on_wait_finally, + ) + + async def _submit_compile_job(self, *, job: CompileJob, zmq_client, loop) -> str: + response = await self._submit_compile_request( + zmq_client=zmq_client, + loop=loop, + plate_path=job.plate_path, + definition_pipeline=job.definition_pipeline, + pipeline_config=job.pipeline_config, + ) + return response["execution_id"] + + async def _wait_compile_job( + self, *, submission_id: str, job: CompileJob, zmq_client, loop + ) -> None: + await self._wait_for_compile_completion( + zmq_client=zmq_client, + loop=loop, + execution_id=submission_id, + plate_path=job.plate_path, + ) + + def _mark_execution_compile_failed(self, plate_path: str, error: Exception) -> None: + logger.error( + "Compile-before-execution failed for %s: %s", + plate_path, + error, + exc_info=True, + ) + self.host.execution_runtime.mark_terminal( + plate_path, TerminalExecutionStatus.FAILED + ) + self.host.emit_error(f"Compile failed for {plate_path}: {error}") + self.host.update_item_list() + + async def _submit_compile_request( + self, + *, + zmq_client, + loop, + plate_path: str, + definition_pipeline: List, + pipeline_config, + ) -> Dict[str, Any]: + def _submit_compile() -> Dict[str, Any]: + logger.info( + "Submit compile: plate=%s steps=%d fingerprint=%s", + plate_path, + len(definition_pipeline), + self._pipeline_fingerprint(definition_pipeline), + ) + return zmq_client.submit_compile( + plate_id=plate_path, + pipeline_steps=definition_pipeline, + global_config=self.host.global_config, + pipeline_config=pipeline_config, + ) + + response = await self._run_blocking(loop, _submit_compile) + if response.get("status") != "accepted": + raise RuntimeError( + f"Compile submission failed for {plate_path}: " + f"{response.get('message', 'Unknown error')}" + ) + execution_id = response.get("execution_id") + if not execution_id: + raise RuntimeError( + f"Compile submission missing execution_id for {plate_path}" + ) + return {"execution_id": execution_id, "response": response} + + async def _wait_for_compile_completion( + self, + *, + zmq_client, + loop, + execution_id: str, + plate_path: str, + ) -> None: + wait_result = await self._run_blocking( + loop, + lambda: zmq_client.wait_for_completion(execution_id), + ) + if wait_result.get("status") != "complete": + raise RuntimeError( + f"Compilation failed for {plate_path}: " + f"{wait_result.get('message', 'Unknown error')}" + ) + + def _build_run_spec(self, plate_path: str) -> Dict[str, Any]: + plate_path = str(plate_path) + definition_pipeline = self.host.get_pipeline_definition(plate_path) + if not definition_pipeline: + logger.warning( + "No pipeline defined for %s, using empty pipeline", + plate_path, + ) + definition_pipeline = [] + self._validate_pipeline_steps(definition_pipeline) + pipeline_config = resolve_pipeline_config_for_plate(self.host, plate_path) + return { + "plate_path": plate_path, + "definition_pipeline": definition_pipeline, + "global_config": self.host.global_config, + "pipeline_config": pipeline_config, + } + + async def _submit_plate( + self, run_spec: Dict[str, Any], compile_artifact_id: str, loop + ) -> None: + if self.client_service.zmq_client is None: + raise RuntimeError("ZMQ client is not connected") + plate_path = run_spec["plate_path"] + definition_pipeline = run_spec["definition_pipeline"] + logger.info("Executing plate: %s", plate_path) + logger.info( + "Submit run: plate=%s artifact_id=%s steps=%d fingerprint=%s", + plate_path, + compile_artifact_id, + len(definition_pipeline), + self._pipeline_fingerprint(definition_pipeline), + ) + + def _submit() -> Dict[str, Any]: + return self.client_service.zmq_client.submit_pipeline( + plate_id=plate_path, + pipeline_steps=definition_pipeline, + global_config=run_spec["global_config"], + pipeline_config=run_spec["pipeline_config"], + compile_artifact_id=compile_artifact_id, + ) + + response = await self._run_blocking(loop, _submit) + + execution_id = response.get("execution_id") + if execution_id: + self.host.plate_execution_ids[plate_path] = execution_id + self.host.current_execution_id = execution_id + + status = response.get("status") + if status == "accepted": + self.host.emit_status(f"Submitted {plate_path} (queued on server)") + if execution_id: + self._start_completion_poller(execution_id, plate_path) + return + + error_msg = response.get("message", "Unknown error") + logger.error("Plate %s submission failed: %s", plate_path, error_msg) + self.host.emit_error(f"Submission failed for {plate_path}: {error_msg}") + self.host.execution_runtime.mark_terminal( + plate_path, TerminalExecutionStatus.FAILED + ) + from objectstate import ObjectStateRegistry + + orchestrator = ObjectStateRegistry.get_object(plate_path) + if orchestrator is not None: + orchestrator._state = OrchestratorState.EXEC_FAILED + self.host.emit_orchestrator_state( + plate_path, OrchestratorState.EXEC_FAILED.value + ) + + async def _handle_execution_failure(self, loop) -> None: + from objectstate import ObjectStateRegistry + + for plate_path in tuple(self.host.execution_runtime.active_plates): + self.host.execution_runtime.mark_terminal( + plate_path, TerminalExecutionStatus.FAILED + ) + orchestrator = ObjectStateRegistry.get_object(plate_path) + if orchestrator is not None: + orchestrator._state = OrchestratorState.EXEC_FAILED + self.host.emit_orchestrator_state( + plate_path, OrchestratorState.EXEC_FAILED.value + ) + + self.host.execution_state = ManagerExecutionState.IDLE + await self._disconnect_client(loop) + self.host.current_execution_id = None + self._refresh_host_execution_ui() + + async def _disconnect_client(self, loop) -> None: + if self.client_service.zmq_client is None: + return + try: + await self.client_service.disconnect() + except Exception as error: + logger.warning("Error disconnecting old client: %s", error) + + @staticmethod + async def _run_blocking(loop, func: Callable[[], T]) -> T: + return await loop.run_in_executor(None, func) + + @staticmethod + def _pipeline_fingerprint(definition_pipeline: List) -> str: + import openhcs.serialization.pycodify_formatters # noqa: F401 + from pycodify import Assignment, generate_python_source + + pipeline_code = generate_python_source( + Assignment("pipeline_steps", definition_pipeline), + header="# Edit this pipeline and save to apply changes", + clean_mode=True, + ) + return hashlib.sha256(pipeline_code.encode("utf-8")).hexdigest()[:12] + + @staticmethod + def _pipeline_step_names(definition_pipeline: List) -> List[str]: + return [str(getattr(step, "name", "")) for step in definition_pipeline] + + @staticmethod + def _flush_pending_ui_edits() -> None: + """Commit pending editor widget state before reading pipeline definitions.""" + app = QApplication.instance() + if app is None: + return + focus_widget = app.focusWidget() + if focus_widget is not None: + focus_widget.clearFocus() + app.processEvents(QEventLoop.ProcessEventsFlag.AllEvents) + + def _start_completion_poller(self, execution_id: str, plate_path: str) -> None: + class _ClientDisconnected(RuntimeError): + pass + + def _poll_status(polled_execution_id: str) -> Dict[str, Any]: + if self.client_service.zmq_client is None: + raise _ClientDisconnected("ZMQ client disconnected") + return self.client_service.zmq_client.get_status(polled_execution_id) + + def _on_running(_execution_id: str, _execution_payload: Dict[str, Any]) -> None: + self.host.update_item_list() + self.host.emit_status(f"▶️ Running {plate_path}") + + def _on_terminal( + _execution_id: str, terminal_status: str, execution_payload: Dict[str, Any] + ) -> None: + current_execution_id = self.host.plate_execution_ids.get(plate_path) + if current_execution_id != _execution_id: + logger.info( + "Ignoring stale terminal status for %s: execution_id=%s current=%s", + plate_path, + _execution_id, + current_execution_id, + ) + return + parsed_terminal_status = parse_terminal_status(terminal_status) + + self.host.execution_runtime.mark_terminal( + plate_path, parsed_terminal_status + ) + result = self._build_terminal_result( + terminal_status=parsed_terminal_status.value, + execution_id=_execution_id, + execution_payload=execution_payload, + ) + self.host.notify_plate_completed( + plate_path, parsed_terminal_status.value, result + ) + self._check_all_completed() + + def _on_status_error(_execution_id: str, message: str) -> None: + current_execution_id = self.host.plate_execution_ids.get(plate_path) + if current_execution_id != _execution_id: + logger.info( + "Ignoring stale status error for %s: execution_id=%s current=%s", + plate_path, + _execution_id, + current_execution_id, + ) + return + self.host.execution_runtime.mark_terminal( + plate_path, TerminalExecutionStatus.FAILED + ) + self.host.notify_plate_completed( + plate_path, + TerminalExecutionStatus.FAILED.value, + { + "status": TerminalExecutionStatus.FAILED.value, + "execution_id": _execution_id, + "message": message, + }, + ) + self._check_all_completed() + + def _on_poll_exception(_execution_id: str, error: Exception) -> bool: + if isinstance(error, _ClientDisconnected): + return False + logger.warning("Error polling status for %s: %s", plate_path, error) + return True + + policy = CallbackExecutionStatusPollPolicy( + poll_status_fn=_poll_status, + poll_interval_seconds_value=0.5, + on_running_fn=_on_running, + on_terminal_fn=_on_terminal, + on_status_error_fn=_on_status_error, + on_poll_exception_fn=_on_poll_exception, + ) + + def poll_completion() -> None: + try: + self._execution_status_poller.run(execution_id, policy) + except Exception as error: + logger.error( + "Error in completion poller for %s: %s", + plate_path, + error, + exc_info=True, + ) + self.host.emit_error(f"{plate_path}: {error}") + + threading.Thread(target=poll_completion, daemon=True).start() + + @staticmethod + def _build_terminal_result( + *, + terminal_status: str, + execution_id: str, + execution_payload: Dict[str, Any], + ) -> Dict[str, Any]: + status = parse_terminal_status(terminal_status) + match status: + case TerminalExecutionStatus.COMPLETE: + return BatchWorkflowService._build_complete_terminal_result( + execution_id, execution_payload + ) + case TerminalExecutionStatus.FAILED: + return BatchWorkflowService._build_failed_terminal_result( + execution_id, execution_payload + ) + case TerminalExecutionStatus.CANCELLED: + return BatchWorkflowService._build_cancelled_terminal_result( + execution_id, execution_payload + ) + raise ValueError(f"Unsupported terminal status '{status}'") + + @staticmethod + def _build_complete_terminal_result( + execution_id: str, execution_payload: Dict[str, Any] + ) -> Dict[str, Any]: + results_summary = execution_payload.get("results_summary", {}) or {} + output_plate_root = None + auto_add_output_plate = None + if isinstance(results_summary, dict): + output_plate_root = results_summary.get("output_plate_root") + auto_add_output_plate = results_summary.get( + "auto_add_output_plate_to_plate_manager" + ) + return { + "status": TerminalExecutionStatus.COMPLETE.value, + "execution_id": execution_id, + "results": results_summary, + "output_plate_root": output_plate_root, + "auto_add_output_plate_to_plate_manager": auto_add_output_plate, + } + + @staticmethod + def _build_failed_terminal_result( + execution_id: str, execution_payload: Dict[str, Any] + ) -> Dict[str, Any]: + return { + "status": TerminalExecutionStatus.FAILED.value, + "execution_id": execution_id, + "message": execution_payload.get("error", "Unknown error"), + "traceback": execution_payload.get("traceback", ""), + } + + @staticmethod + def _build_cancelled_terminal_result( + execution_id: str, _execution_payload: Dict[str, Any] + ) -> Dict[str, Any]: + return { + "status": TerminalExecutionStatus.CANCELLED.value, + "execution_id": execution_id, + "message": "Execution was cancelled", + } + + def _coalesced_progress_update(self) -> None: + if self.client_service.zmq_client is not None: + self._server_info_poller.tick() + if not self._progress_dirty: + return + self._progress_dirty = False + self._runtime_projection = build_execution_runtime_projection_from_registry( + self.host._progress_tracker + ) + self.host.set_runtime_progress_projection(self._runtime_projection) + self.host.set_execution_server_info(self._get_server_info_snapshot()) + self._emit_execution_server_status() + self.host.update_item_list() + + def _on_progress(self, message: dict) -> None: + try: + event = ProgressEvent.from_dict(message) + self.host._progress_tracker.register_event(event.execution_id, event) + except Exception as error: + logger.warning("Failed to parse/register progress event: %s", error) + finally: + self._mark_progress_dirty() + + def _on_registry_event(self, _execution_id: str, _event: ProgressEvent) -> None: + """Mark projection dirty when shared registry changes from any producer.""" + self._mark_progress_dirty() + + def _emit_execution_server_status(self) -> None: + status_view = self._server_status_presenter.build_status_text( + projection=self._runtime_projection, + server_info=self._get_server_info_snapshot(), + ) + self.host.emit_status(status_view.text) + + def _get_server_info_snapshot(self) -> ExecutionServerInfo | None: + return self._server_info_poller.get_snapshot_copy() + + def _fetch_server_info_snapshot(self) -> ExecutionServerInfo: + if self.client_service.zmq_client is None: + raise RuntimeError("ZMQ client is not connected") + pong = self.client_service.zmq_client.get_server_info_snapshot() + parsed = self._server_info_parser.parse(pong.to_dict()) + if not isinstance(parsed, ExecutionServerInfo): + raise ValueError( + f"Expected ExecutionServerInfo, got {type(parsed).__name__}" + ) + return parsed + + def _mark_progress_dirty(self) -> None: + self._progress_dirty = True + + def _check_all_completed(self) -> None: + if self.host.execution_state not in ( + ManagerExecutionState.RUNNING, + *STOP_PENDING_MANAGER_STATES, + ): + return + if not self.host.execution_runtime.all_batch_terminal(): + return + completed, failed = self.host.execution_runtime.terminal_counts() + self.host.notify_all_plates_completed(completed, failed) + + def stop_execution(self, force: bool = False) -> None: + port = self.port + + def kill_server() -> None: + try: + # Force-kill is best-effort: the server may already be gone if a graceful + # stop just completed, so treat "not found" style outcomes as success. + plan = ServerKillPlan( + graceful=not force, + strict_failures=not force, + emit_signal_on_failure=force, + success_message=f"Stopped execution server on port {port}", + ) + success, message = self._server_kill_service.kill_ports( + ports=[port], + plan=plan, + on_server_killed=lambda _port: self._emit_cancelled_for_all_plates(), + log_info=logger.info, + log_warning=logger.warning, + log_error=logger.error, + ) + if not success: + if self.host.execution_state in ( + ManagerExecutionState.STOPPING, + ManagerExecutionState.FORCE_KILL_READY, + ManagerExecutionState.IDLE, + ): + logger.info( + "Suppressing stale stop failure while stop is already terminalizing: %s", + message, + ) + self._emit_cancelled_for_all_plates() + return + self.host.emit_error(message) + return + except Exception as error: + logger.error("Error stopping server: %s", error) + self.host.emit_error(f"Error stopping execution: {error}") + + threading.Thread(target=kill_server, daemon=True).start() + + if force: + # Keep UI responsive on force-kill: mark plates cancelled immediately on the + # caller thread while kill work continues in the background. + self._emit_cancelled_for_all_plates() + self.disconnect_async() + + def _emit_cancelled_for_all_plates(self) -> None: + for plate_path in self.host.execution_runtime.cancellable_plates(): + self.host.emit_execution_complete( + {"status": TerminalExecutionStatus.CANCELLED.value}, plate_path + ) + + def disconnect(self) -> None: + if self.client_service.zmq_client is None: + return + try: + self.client_service.disconnect_sync() + except Exception as error: + logger.warning("Error disconnecting ZMQ client: %s", error) + + def disconnect_async(self) -> None: + """Disconnect client on a background thread to avoid UI stalls.""" + + def _disconnect() -> None: + self.disconnect() + + threading.Thread(target=_disconnect, daemon=True).start() + + def _refresh_host_execution_ui(self) -> None: + refresh_fn = getattr(self.host, "refresh_execution_ui", None) + if callable(refresh_fn): + refresh_fn() + return + self.host.update_item_list() + self.host.update_button_states() + + def _validate_pipeline_steps(self, pipeline: List) -> None: + for step in pipeline: + if step.func is None: + raise AttributeError( + f"Step '{step.name}' has func=None. " + "This usually means the pipeline was loaded from a compiled state." + ) + + @staticmethod + def _set_orchestrator_state(plate_path: str, state: OrchestratorState) -> None: + from objectstate import ObjectStateRegistry + + orchestrator = ObjectStateRegistry.get_object(plate_path) + if orchestrator is not None: + orchestrator._state = state + + def _handle_compile_failure( + self, plate_data: Dict[str, Any], plate_path: str, error: Exception + ) -> None: + logger.error("COMPILATION ERROR: %s: %s", plate_path, error, exc_info=True) + plate_data["error"] = str(error) + self.host.clear_plate_execution_tracking(plate_path) + self._set_orchestrator_state(plate_path, OrchestratorState.COMPILE_FAILED) + self.host.plate_compile_pending.discard(plate_path) + self.host.update_item_list() + self.host.emit_orchestrator_state(plate_path, "COMPILE_FAILED") + self.host.emit_compilation_error(plate_data["name"], str(error)) + + def _reset_progress_for_new_batch(self) -> None: + self._runtime_projection = reset_progress_views_for_new_batch( + self.host, + projection=ExecutionRuntimeProjection(), + ) + self._server_info_poller.reset() + self._mark_progress_dirty() diff --git a/openhcs/pyqt_gui/widgets/shared/services/compilation_service.py b/openhcs/pyqt_gui/widgets/shared/services/compilation_service.py deleted file mode 100644 index be41a2ce1..000000000 --- a/openhcs/pyqt_gui/widgets/shared/services/compilation_service.py +++ /dev/null @@ -1,212 +0,0 @@ -""" -Compilation service for plate pipeline compilation. - -Extracts compilation logic from PlateManagerWidget into a reusable service. -""" -import copy -import asyncio -import logging -from typing import Dict, List, Any, Protocol, runtime_checkable - -from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator -from openhcs.core.pipeline import Pipeline -from openhcs.constants.constants import VariableComponents - -logger = logging.getLogger(__name__) - - -@runtime_checkable -class CompilationHost(Protocol): - """Protocol for widgets that host the compilation service.""" - - # State attributes the service needs access to - global_config: Any - # NOTE: orchestrators now accessed via ObjectStateRegistry.get_object(plate_path) - plate_configs: Dict[str, Dict] - plate_compiled_data: Dict[str, Any] - - # Methods the service calls back to - def emit_progress_started(self, count: int) -> None: ... - def emit_progress_updated(self, value: int) -> None: ... - def emit_progress_finished(self) -> None: ... - def emit_orchestrator_state(self, plate_path: str, state: str) -> None: ... - def emit_compilation_error(self, plate_name: str, error: str) -> None: ... - def emit_status(self, msg: str) -> None: ... - def get_pipeline_definition(self, plate_path: str) -> List: ... - def update_button_states(self) -> None: ... - - -class CompilationService: - """ - Service for compiling plate pipelines. - - Handles: - - Orchestrator initialization - - Pipeline compilation with context setup - - Progress reporting via host callbacks - """ - - def __init__(self, host: CompilationHost): - self.host = host - - async def compile_plates(self, selected_items: List[Dict]) -> None: - """ - Compile pipelines for selected plates. - - Args: - selected_items: List of plate data dicts with 'path' and 'name' keys - """ - # Set up global context in worker thread - from openhcs.config_framework.lazy_factory import ensure_global_config_context - from openhcs.core.config import GlobalPipelineConfig - ensure_global_config_context(GlobalPipelineConfig, self.host.global_config) - - self.host.emit_progress_started(len(selected_items)) - - for i, plate_data in enumerate(selected_items): - plate_path = plate_data['path'] - - # Get definition pipeline - definition_pipeline = self.host.get_pipeline_definition(plate_path) - if not definition_pipeline: - logger.warning(f"No pipeline defined for {plate_data['name']}, using empty pipeline") - definition_pipeline = [] - - # Validate func attributes - self._validate_pipeline_steps(definition_pipeline) - - try: - # Get or create orchestrator - orchestrator = await self._get_or_create_orchestrator(plate_path) - - # Make fresh copy for compilation - execution_pipeline = copy.deepcopy(definition_pipeline) - self._fix_step_ids(execution_pipeline) - - # Compile - compiled_data = await self._compile_pipeline( - orchestrator, definition_pipeline, execution_pipeline - ) - - # Store results - self.host.plate_compiled_data[plate_path] = compiled_data - logger.info(f"Successfully compiled {plate_path}") - self.host.emit_orchestrator_state(plate_path, "COMPILED") - - except Exception as e: - logger.error(f"COMPILATION ERROR: {plate_path}: {e}", exc_info=True) - plate_data['error'] = str(e) - self.host.emit_orchestrator_state(plate_path, "COMPILE_FAILED") - self.host.emit_compilation_error(plate_data['name'], str(e)) - - self.host.emit_progress_updated(i + 1) - - self.host.emit_progress_finished() - self.host.emit_status(f"Compilation completed for {len(selected_items)} plate(s)") - self.host.update_button_states() - - def _validate_pipeline_steps(self, pipeline: List) -> None: - """Validate that steps have required func attribute.""" - for i, step in enumerate(pipeline): - if not hasattr(step, 'func'): - raise AttributeError( - f"Step '{step.name}' is missing 'func' attribute. " - "This usually means the pipeline was loaded from a compiled state." - ) - - async def _get_or_create_orchestrator(self, plate_path: str) -> PipelineOrchestrator: - """Get existing orchestrator or create and initialize a new one.""" - from openhcs.config_framework.lazy_factory import ensure_global_config_context - from openhcs.core.config import GlobalPipelineConfig - from objectstate import ObjectStateRegistry, ObjectState - - orchestrator = ObjectStateRegistry.get_object(plate_path) - if orchestrator is not None: - if not orchestrator.is_initialized(): - def initialize_with_context(): - ensure_global_config_context(GlobalPipelineConfig, self.host.global_config) - return orchestrator.initialize() - - loop = asyncio.get_event_loop() - await loop.run_in_executor(None, initialize_with_context) - else: - # Create new orchestrator with isolated registry - from polystore.base import _create_storage_registry - plate_registry = _create_storage_registry() - - orchestrator = PipelineOrchestrator( - plate_path=plate_path, - storage_registry=plate_registry - ) - - saved_config = self.host.plate_configs.get(str(plate_path)) - if saved_config: - orchestrator.apply_pipeline_config(saved_config) - - def initialize_with_context(): - ensure_global_config_context(GlobalPipelineConfig, self.host.global_config) - return orchestrator.initialize() - - loop = asyncio.get_event_loop() - await loop.run_in_executor(None, initialize_with_context) - - # Register orchestrator in ObjectState (single source of truth) - orchestrator_state = ObjectState( - object_instance=orchestrator, - scope_id=str(plate_path), - parent_state=ObjectStateRegistry.get_by_scope(""), - ) - ObjectStateRegistry.register(orchestrator_state) - - return orchestrator - - def _fix_step_ids(self, pipeline: List) -> None: - """Fix step IDs after deep copy and ensure variable_components.""" - from dataclasses import replace - - for step in pipeline: - step.step_id = str(id(step)) - - # Ensure variable_components is never None - if step.processing_config.variable_components is None or not step.processing_config.variable_components: - if step.processing_config.variable_components is None: - logger.warning(f"Step '{step.name}' has None variable_components, setting default") - step.processing_config = replace( - step.processing_config, - variable_components=[VariableComponents.SITE] - ) - - async def _compile_pipeline( - self, - orchestrator: PipelineOrchestrator, - definition_pipeline: List, - execution_pipeline: List - ) -> Dict: - """Compile pipeline and return compiled data dict.""" - from openhcs.config_framework.lazy_factory import ensure_global_config_context - from openhcs.core.config import GlobalPipelineConfig - from openhcs.constants import MULTIPROCESSING_AXIS - - pipeline_obj = Pipeline(steps=execution_pipeline) - loop = asyncio.get_event_loop() - - # Get wells - wells = await loop.run_in_executor( - None, - lambda: orchestrator.get_component_keys(MULTIPROCESSING_AXIS) - ) - - # Compile with context - def compile_with_context(): - ensure_global_config_context(GlobalPipelineConfig, self.host.global_config) - return orchestrator.compile_pipelines(pipeline_obj.steps, wells) - - compilation_result = await loop.run_in_executor(None, compile_with_context) - compiled_contexts = compilation_result['compiled_contexts'] - - return { - 'definition_pipeline': definition_pipeline, - 'execution_pipeline': execution_pipeline, - 'compiled_contexts': compiled_contexts - } - diff --git a/openhcs/pyqt_gui/widgets/shared/services/execution_server_status_presenter.py b/openhcs/pyqt_gui/widgets/shared/services/execution_server_status_presenter.py new file mode 100644 index 000000000..302fd7b17 --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/services/execution_server_status_presenter.py @@ -0,0 +1,52 @@ +"""Execution-server status line presenter for batch workflow UI.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List + +from openhcs.core.progress.projection import ExecutionRuntimeProjection +from pyqt_reactive.services import ( + ExecutionServerInfo, +) + + +@dataclass(frozen=True) +class ExecutionServerStatusView: + """Rendered status text for plate-manager status bar.""" + + text: str + + +class ExecutionServerStatusPresenter: + """Build status text from runtime projection.""" + + def build_status_text( + self, + *, + projection: ExecutionRuntimeProjection, + server_info: ExecutionServerInfo | None, + ) -> ExecutionServerStatusView: + plate_count = len(projection.by_plate_latest) + if plate_count == 0: + return ExecutionServerStatusView(text="Ready") + + parts: List[str] = [] + if projection.compiling_count > 0: + parts.append(f"⏳ {projection.compiling_count} compiling") + if projection.executing_count > 0: + parts.append(f"⚙️ {projection.executing_count} executing") + if projection.compiled_count > 0: + parts.append(f"✓ {projection.compiled_count} compiled") + if projection.complete_count > 0: + parts.append(f"✅ {projection.complete_count} complete") + if projection.failed_count > 0: + parts.append(f"❌ {projection.failed_count} failed") + + status_text = ", ".join(parts) if parts else "idle" + return ExecutionServerStatusView( + text=( + f"Server: {status_text} | " + f"{plate_count} plates | avg {projection.overall_percent:.1f}%" + ) + ) diff --git a/openhcs/pyqt_gui/widgets/shared/services/execution_state.py b/openhcs/pyqt_gui/widgets/shared/services/execution_state.py new file mode 100644 index 000000000..d912c4eb1 --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/services/execution_state.py @@ -0,0 +1,155 @@ +"""Execution state enums for Plate Manager workflow.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Iterable + +from openhcs.core.orchestrator.orchestrator import OrchestratorState + + +class ManagerExecutionState(str, Enum): + IDLE = "idle" + RUNNING = "running" + STOPPING = "stopping" + FORCE_KILL_READY = "force_kill_ready" + + +class TerminalExecutionStatus(str, Enum): + COMPLETE = "complete" + FAILED = "failed" + CANCELLED = "cancelled" + + +@dataclass(frozen=True) +class TerminalUiPolicy: + orchestrator_state: OrchestratorState + status_prefix: str | None = None + emit_failure: bool = False + auto_add_output_plate: bool = False + + +@dataclass +class ExecutionBatchRuntime: + """Tracks current run-attempt terminal outcomes and active plates.""" + + batch_plates: tuple[str, ...] = () + active_plates: set[str] = field(default_factory=set) + terminal_status_by_plate: dict[str, TerminalExecutionStatus] = field( + default_factory=dict + ) + + def begin_batch(self, plate_paths: Iterable[str]) -> None: + ordered = tuple(dict.fromkeys(str(path) for path in plate_paths)) + self.batch_plates = ordered + self.active_plates = set(ordered) + self.terminal_status_by_plate.clear() + + def mark_terminal( + self, plate_path: str, status: str | TerminalExecutionStatus + ) -> None: + terminal_status = parse_terminal_status(status) + self.terminal_status_by_plate[plate_path] = terminal_status + self.active_plates.discard(plate_path) + + def is_active(self, plate_path: str) -> bool: + return plate_path in self.active_plates + + def terminal_status(self, plate_path: str) -> TerminalExecutionStatus | None: + return self.terminal_status_by_plate.get(plate_path) + + def clear_plate(self, plate_path: str, *, clear_terminal: bool = True) -> None: + self.active_plates.discard(plate_path) + if clear_terminal: + self.terminal_status_by_plate.pop(plate_path, None) + + def clear_batch(self) -> None: + self.batch_plates = () + self.active_plates.clear() + self.terminal_status_by_plate.clear() + + def all_batch_terminal(self) -> bool: + if not self.batch_plates: + return True + return all( + plate_path in self.terminal_status_by_plate + for plate_path in self.batch_plates + ) + + def terminal_counts(self) -> tuple[int, int]: + statuses = [ + self.terminal_status_by_plate[plate_path] + for plate_path in self.batch_plates + if plate_path in self.terminal_status_by_plate + ] + completed = sum( + 1 for status in statuses if status == TerminalExecutionStatus.COMPLETE + ) + failed = sum( + 1 + for status in statuses + if status + in (TerminalExecutionStatus.FAILED, TerminalExecutionStatus.CANCELLED) + ) + return completed, failed + + def cancellable_plates(self) -> tuple[str, ...]: + return tuple( + plate_path + for plate_path in self.batch_plates + if plate_path in self.active_plates + ) + + +STOP_PENDING_MANAGER_STATES = frozenset( + { + ManagerExecutionState.STOPPING, + ManagerExecutionState.FORCE_KILL_READY, + } +) +BUSY_MANAGER_STATES = frozenset( + { + ManagerExecutionState.RUNNING, + ManagerExecutionState.STOPPING, + ManagerExecutionState.FORCE_KILL_READY, + } +) + + +TERMINAL_UI_POLICIES = { + TerminalExecutionStatus.COMPLETE: TerminalUiPolicy( + orchestrator_state=OrchestratorState.COMPLETED, + status_prefix="✓ Completed", + auto_add_output_plate=True, + ), + TerminalExecutionStatus.FAILED: TerminalUiPolicy( + orchestrator_state=OrchestratorState.EXEC_FAILED, + emit_failure=True, + ), + TerminalExecutionStatus.CANCELLED: TerminalUiPolicy( + orchestrator_state=OrchestratorState.READY, + status_prefix="✗ Cancelled", + ), +} + + +_TERMINAL_STATUS_ALIASES: dict[str, TerminalExecutionStatus] = { + "error": TerminalExecutionStatus.FAILED, +} + + +def parse_terminal_status( + status: str | TerminalExecutionStatus, +) -> TerminalExecutionStatus: + if isinstance(status, TerminalExecutionStatus): + return status + alias = _TERMINAL_STATUS_ALIASES.get(status) + if alias is not None: + return alias + return TerminalExecutionStatus(status) + + +def terminal_ui_policy(status: str | TerminalExecutionStatus) -> TerminalUiPolicy: + terminal = parse_terminal_status(status) + return TERMINAL_UI_POLICIES[terminal] diff --git a/openhcs/pyqt_gui/widgets/shared/services/plate_config_resolver.py b/openhcs/pyqt_gui/widgets/shared/services/plate_config_resolver.py new file mode 100644 index 000000000..cc9d78000 --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/services/plate_config_resolver.py @@ -0,0 +1,30 @@ +"""Shared helpers for resolving per-plate pipeline configuration.""" + +from __future__ import annotations + + +def resolve_pipeline_config_for_plate(host, plate_path: str): + """Resolve per-plate PipelineConfig from canonical sources. + + Resolution order: + 1) ObjectState scope (authoritative editable config) + 2) live orchestrator object + 3) host.plate_configs fallback + 4) default PipelineConfig() + """ + from objectstate import ObjectStateRegistry + from openhcs.core.config import PipelineConfig + + state = ObjectStateRegistry.get_by_scope(plate_path) + if state is not None: + return state.to_object(update_delegate=False) + + orchestrator = ObjectStateRegistry.get_object(plate_path) + if orchestrator is not None: + return orchestrator.pipeline_config + + saved_config = host.plate_configs.get(str(plate_path)) + if saved_config: + return saved_config + + return PipelineConfig() diff --git a/openhcs/pyqt_gui/widgets/shared/services/plate_status_presenter.py b/openhcs/pyqt_gui/widgets/shared/services/plate_status_presenter.py new file mode 100644 index 000000000..203593797 --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/services/plate_status_presenter.py @@ -0,0 +1,80 @@ +"""Plate list status presenter for Plate Manager rows.""" + +from __future__ import annotations + +from typing import Optional + +from openhcs.core.orchestrator.orchestrator import OrchestratorState +from openhcs.core.progress.projection import PlateRuntimeProjection, PlateRuntimeState +from openhcs.pyqt_gui.widgets.shared.services.execution_state import ( + TerminalExecutionStatus, +) + + +class PlateStatusPresenter: + """Render deterministic plate status text from execution + orchestrator signals.""" + + TERMINAL_LABELS = { + TerminalExecutionStatus.CANCELLED: "✗ Cancelled", + TerminalExecutionStatus.FAILED: "❌ Exec Failed", + TerminalExecutionStatus.COMPLETE: "✅ Complete", + } + ORCHESTRATOR_LABELS = { + OrchestratorState.READY: "✓ Init", + OrchestratorState.COMPILED: "✓ Compiled", + OrchestratorState.COMPLETED: "✅ Complete", + OrchestratorState.INIT_FAILED: "❌ Init Failed", + OrchestratorState.COMPILE_FAILED: "❌ Compile Failed", + OrchestratorState.EXEC_FAILED: "❌ Exec Failed", + OrchestratorState.EXECUTING: "🔄 Executing", + } + RUNTIME_STATE_LABELS = { + PlateRuntimeState.COMPILING: "⏳ Compiling", + PlateRuntimeState.COMPILED: "✅ Compiled", + PlateRuntimeState.FAILED: "❌ Failed", + PlateRuntimeState.COMPLETE: "✅ Complete", + PlateRuntimeState.EXECUTING: "⚙️ Executing", + } + + @classmethod + def build_status_prefix( + cls, + *, + orchestrator_state: Optional[OrchestratorState], + is_init_pending: bool, + is_compile_pending: bool, + is_execution_active: bool, + terminal_status: Optional[TerminalExecutionStatus], + queue_position: Optional[int], + runtime_projection: Optional[PlateRuntimeProjection], + ) -> str: + # Runtime projection is canonical whenever present. + if runtime_projection is not None: + return cls._format_runtime_plate_status(runtime_projection) + + if queue_position is not None: + return f"⏳ Queued 0.0% (q#{queue_position})" + + if is_execution_active: + return "⏳ Pending" + + if is_init_pending: + return "⏳ Init" + if is_compile_pending: + return "⏳ Compile" + + if terminal_status is not None: + terminal_label = cls.TERMINAL_LABELS.get(terminal_status) + if terminal_label is not None: + return terminal_label + + if orchestrator_state is None: + return "" + return cls.ORCHESTRATOR_LABELS.get(orchestrator_state, "") + + @staticmethod + def _format_runtime_plate_status(plate: PlateRuntimeProjection) -> str: + label = PlateStatusPresenter.RUNTIME_STATE_LABELS.get(plate.state) + if label is None: + return "" + return f"{label} {plate.percent:.1f}%" diff --git a/openhcs/pyqt_gui/widgets/shared/services/progress_batch_reset.py b/openhcs/pyqt_gui/widgets/shared/services/progress_batch_reset.py new file mode 100644 index 000000000..3051e78b1 --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/services/progress_batch_reset.py @@ -0,0 +1,26 @@ +"""Shared progress reset utilities for new compile/execute batches.""" + +from __future__ import annotations + +from openhcs.core.progress.projection import ExecutionRuntimeProjection + + +def reset_progress_views_for_new_batch( + host, projection: ExecutionRuntimeProjection | None = None +) -> ExecutionRuntimeProjection: + """Clear stale execution progress and reset host-facing projections. + + This removes all prior execution snapshots from the shared tracker so each + new batch starts with a clean subtree. + """ + runtime_projection = projection or ExecutionRuntimeProjection() + + execution_ids = list(host._progress_tracker.get_execution_ids()) + for execution_id in execution_ids: + host._progress_tracker.clear_execution(execution_id) + + host.set_runtime_progress_projection(runtime_projection) + host.set_execution_server_info(None) + host.update_item_list() + return runtime_projection + diff --git a/openhcs/pyqt_gui/widgets/shared/services/zmq_client_service.py b/openhcs/pyqt_gui/widgets/shared/services/zmq_client_service.py new file mode 100644 index 000000000..7f56520fa --- /dev/null +++ b/openhcs/pyqt_gui/widgets/shared/services/zmq_client_service.py @@ -0,0 +1,65 @@ +""" +Shared ZMQ client manager for compile/run flows. +""" + +from __future__ import annotations + +import asyncio +import logging + +logger = logging.getLogger(__name__) + + +class ZMQClientService: + """Create/connect/disconnect a ZMQ execution client.""" + + def __init__(self, port: int = 7777): + self.port = port + self.zmq_client = None + + async def connect( + self, + progress_callback=None, + persistent: bool = True, + timeout: float = 15, + ): + """Create a client and connect to the execution server.""" + from openhcs.runtime.zmq_execution_client import ZMQExecutionClient + + if self.zmq_client is not None and self.zmq_client.is_connected(): + existing_callback = self.zmq_client.progress_callback + if existing_callback is progress_callback: + return self.zmq_client + await self.disconnect() + loop = asyncio.get_event_loop() + self.zmq_client = ZMQExecutionClient( + port=self.port, + persistent=persistent, + progress_callback=progress_callback, + ) + connected = await loop.run_in_executor( + None, lambda: self.zmq_client.connect(timeout=timeout) + ) + if not connected: + self.zmq_client = None + raise RuntimeError("Failed to connect to ZMQ execution server") + logger.info("✅ Connected to ZMQ execution server") + return self.zmq_client + + async def disconnect(self) -> None: + """Disconnect the client (async-safe).""" + if self.zmq_client is None: + return + loop = asyncio.get_event_loop() + client = self.zmq_client + self.zmq_client = None + await loop.run_in_executor(None, client.disconnect) + + def disconnect_sync(self) -> None: + """Disconnect the client (sync).""" + if self.zmq_client is None: + return + try: + self.zmq_client.disconnect() + finally: + self.zmq_client = None diff --git a/openhcs/pyqt_gui/widgets/shared/services/zmq_execution_service.py b/openhcs/pyqt_gui/widgets/shared/services/zmq_execution_service.py deleted file mode 100644 index 128168102..000000000 --- a/openhcs/pyqt_gui/widgets/shared/services/zmq_execution_service.py +++ /dev/null @@ -1,321 +0,0 @@ -""" -ZMQ Execution Service - Manages ZMQ client lifecycle and plate execution. - -Extracted from PlateManagerWidget to reduce widget complexity. -The service handles: -- ZMQ client connection/disconnection -- Pipeline submission to ZMQ server -- Execution polling and status tracking -- Graceful/force shutdown -""" - -import logging -import threading -import asyncio -from typing import Dict, Optional, Callable, Any, Protocol, List - -from openhcs.core.orchestrator.orchestrator import OrchestratorState - -logger = logging.getLogger(__name__) - - -class ExecutionHost(Protocol): - """Protocol for the widget that hosts ZMQ execution.""" - - # State attributes - execution_state: str - plate_execution_ids: Dict[str, str] - plate_execution_states: Dict[str, str] - # NOTE: orchestrators now accessed via ObjectStateRegistry.get_object(plate_path) - plate_compiled_data: Dict[str, Any] - global_config: Any - current_execution_id: Optional[str] - - # Signal emission methods - def emit_status(self, msg: str) -> None: ... - def emit_error(self, msg: str) -> None: ... - def emit_orchestrator_state(self, plate_path: str, state: str) -> None: ... - def emit_execution_complete(self, result: dict, plate_path: str) -> None: ... - def emit_clear_logs(self) -> None: ... - def update_button_states(self) -> None: ... - def update_item_list(self) -> None: ... - - # Execution completion hooks - def on_plate_completed(self, plate_path: str, status: str, result: dict) -> None: ... - def on_all_plates_completed(self, completed_count: int, failed_count: int) -> None: ... - - -class ZMQExecutionService: - """ - Service for managing ZMQ execution of pipelines. - - Handles client lifecycle, submission, polling, and shutdown. - Delegates UI updates back to host widget via signals. - """ - - def __init__(self, host: ExecutionHost, port: int = 7777): - self.host = host - self.port = port - self.zmq_client = None - - async def run_plates(self, ready_items: List[Dict]) -> None: - """Run plates using ZMQ execution client.""" - try: - from openhcs.runtime.zmq_execution_client import ZMQExecutionClient - - plate_paths = [item['path'] for item in ready_items] - logger.info(f"Starting ZMQ execution for {len(plate_paths)} plates") - - self.host.emit_clear_logs() - loop = asyncio.get_event_loop() - - # Cleanup old client - await self._disconnect_client(loop) - - # Create new client - logger.info("🔌 Creating new ZMQ client") - self.zmq_client = ZMQExecutionClient( - port=self.port, - persistent=True, - progress_callback=self._on_progress - ) - - # Connect - connected = await loop.run_in_executor(None, lambda: self.zmq_client.connect(timeout=15)) - if not connected: - raise RuntimeError("Failed to connect to ZMQ execution server") - logger.info("✅ Connected to ZMQ execution server") - - # Initialize execution tracking - self.host.plate_execution_ids.clear() - self.host.plate_execution_states.clear() - - from objectstate import ObjectStateRegistry - for item in ready_items: - plate_path = item['path'] - self.host.plate_execution_states[plate_path] = "queued" - orchestrator = ObjectStateRegistry.get_object(plate_path) - if orchestrator is not None: - orchestrator._state = OrchestratorState.EXECUTING - self.host.emit_orchestrator_state(plate_path, OrchestratorState.EXECUTING.value) - - self.host.execution_state = "running" - self.host.emit_status(f"Submitting {len(ready_items)} plate(s) to ZMQ server...") - self.host.update_button_states() - - # Submit each plate - for plate_path in plate_paths: - await self._submit_plate(plate_path, loop) - - except Exception as e: - logger.error(f"Failed to execute plates via ZMQ: {e}", exc_info=True) - self.host.emit_error(f"Failed to execute: {e}") - await self._handle_execution_failure(loop) - - async def _disconnect_client(self, loop) -> None: - """Disconnect existing ZMQ client if any.""" - if self.zmq_client is not None: - logger.info("🧹 Disconnecting previous ZMQ client") - try: - await loop.run_in_executor(None, self.zmq_client.disconnect) - except Exception as e: - logger.warning(f"Error disconnecting old client: {e}") - finally: - self.zmq_client = None - - async def _submit_plate(self, plate_path: str, loop) -> None: - """Submit a single plate for execution.""" - compiled_data = self.host.plate_compiled_data[plate_path] - definition_pipeline = compiled_data['definition_pipeline'] - - # Get config - from objectstate import ObjectStateRegistry - orchestrator = ObjectStateRegistry.get_object(plate_path) - if orchestrator is not None: - global_config = self.host.global_config - pipeline_config = orchestrator.pipeline_config - else: - global_config = self.host.global_config - from openhcs.core.config import PipelineConfig - pipeline_config = PipelineConfig() - - logger.info(f"Executing plate: {plate_path}") - - def _submit(): - return self.zmq_client.submit_pipeline( - plate_id=str(plate_path), - pipeline_steps=definition_pipeline, - global_config=global_config, - pipeline_config=pipeline_config - ) - - response = await loop.run_in_executor(None, _submit) - - execution_id = response.get('execution_id') - if execution_id: - self.host.plate_execution_ids[plate_path] = execution_id - self.host.current_execution_id = execution_id - - logger.info(f"Plate {plate_path} submission response: {response.get('status')}") - - status = response.get('status') - if status == 'accepted': - logger.info(f"Plate {plate_path} execution submitted successfully, ID={execution_id}") - self.host.emit_status(f"Submitted {plate_path} (queued on server)") - if execution_id: - self._start_completion_poller(execution_id, plate_path) - else: - error_msg = response.get('message', 'Unknown error') - logger.error(f"Plate {plate_path} submission failed: {error_msg}") - self.host.emit_error(f"Submission failed for {plate_path}: {error_msg}") - self.host.plate_execution_states[plate_path] = "failed" - from objectstate import ObjectStateRegistry - orchestrator = ObjectStateRegistry.get_object(plate_path) - if orchestrator is not None: - orchestrator._state = OrchestratorState.EXEC_FAILED - self.host.emit_orchestrator_state(plate_path, OrchestratorState.EXEC_FAILED.value) - - async def _handle_execution_failure(self, loop) -> None: - """Handle execution failure - mark plates and cleanup.""" - from objectstate import ObjectStateRegistry - for plate_path in self.host.plate_execution_states.keys(): - self.host.plate_execution_states[plate_path] = "failed" - orchestrator = ObjectStateRegistry.get_object(plate_path) - if orchestrator is not None: - orchestrator._state = OrchestratorState.EXEC_FAILED - self.host.emit_orchestrator_state(plate_path, OrchestratorState.EXEC_FAILED.value) - - self.host.execution_state = "idle" - await self._disconnect_client(loop) - self.host.current_execution_id = None - self.host.update_button_states() - - def _start_completion_poller(self, execution_id: str, plate_path: str) -> None: - """Start background thread to poll for plate execution completion.""" - import time - - def poll_completion(): - try: - previous_status = "queued" - while True: - time.sleep(0.5) - - if self.zmq_client is None: - logger.debug(f"ZMQ client disconnected, stopping poller for {plate_path}") - break - - try: - status_response = self.zmq_client.get_status(execution_id) - - if status_response.get('status') == 'ok': - execution = status_response.get('execution', {}) - exec_status = execution.get('status') - - # Detect queued → running transition - if exec_status == 'running' and previous_status == 'queued': - logger.info(f"🔄 Detected transition: {plate_path} queued → running") - self.host.plate_execution_states[plate_path] = "running" - self.host.update_item_list() - self.host.emit_status(f"▶️ Running {plate_path}") - previous_status = "running" - - # Check completion - if exec_status == 'complete': - logger.info(f"✅ Execution complete: {plate_path}") - result = {'status': 'complete', 'execution_id': execution_id, - 'results': execution.get('results_summary', {})} - self.host.on_plate_completed(plate_path, 'complete', result) - self._check_all_completed() - break - elif exec_status == 'failed': - error_msg = execution.get('error', 'Unknown error') - traceback_str = execution.get('traceback', '') - - # Log full traceback if available - if traceback_str: - logger.error(f"❌ Execution failed: {plate_path}\n{traceback_str}") - else: - logger.error(f"❌ Execution failed: {plate_path}: {error_msg}") - - result = {'status': 'error', 'execution_id': execution_id, - 'message': error_msg, 'traceback': traceback_str} - self.host.on_plate_completed(plate_path, 'failed', result) - self._check_all_completed() - break - elif exec_status == 'cancelled': - logger.info(f"🚫 Execution cancelled: {plate_path}") - result = {'status': 'cancelled', 'execution_id': execution_id, - 'message': 'Execution was cancelled'} - self.host.on_plate_completed(plate_path, 'cancelled', result) - self._check_all_completed() - break - except Exception as poll_error: - logger.warning(f"Error polling status for {plate_path}: {poll_error}") - - except Exception as e: - logger.error(f"Error in completion poller for {plate_path}: {e}", exc_info=True) - self.host.emit_error(f"{plate_path}: {e}") - - thread = threading.Thread(target=poll_completion, daemon=True) - thread.start() - - def _on_progress(self, message: dict) -> None: - """Handle progress updates from ZMQ execution server.""" - try: - well_id = message.get('well_id', 'unknown') - step = message.get('step', 'unknown') - status = message.get('status', 'unknown') - self.host.emit_status(f"[{well_id}] {step}: {status}") - except Exception as e: - logger.warning(f"Failed to handle progress update: {e}") - - def _check_all_completed(self) -> None: - """Check if all plates are completed and call host hook if so.""" - all_done = all( - state in ("completed", "failed") - for state in self.host.plate_execution_states.values() - ) - if all_done: - logger.info("All plates completed") - completed = sum(1 for s in self.host.plate_execution_states.values() if s == "completed") - failed = sum(1 for s in self.host.plate_execution_states.values() if s == "failed") - self.host.on_all_plates_completed(completed, failed) - - def stop_execution(self, force: bool = False) -> None: - """Stop execution - graceful or force kill.""" - if self.zmq_client is None: - return - - port = self.port - - def kill_server(): - from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG - from zmqruntime.client import ZMQClient - try: - graceful = not force - logger.info(f"🛑 {'Gracefully' if graceful else 'Force'} killing server on port {port}...") - success = ZMQClient.kill_server_on_port(port, graceful=graceful, config=OPENHCS_ZMQ_CONFIG) - - if success: - logger.info(f"✅ Successfully {'quit' if graceful else 'force killed'} server") - for plate_path in list(self.host.plate_execution_states.keys()): - self.host.emit_execution_complete({'status': 'cancelled'}, plate_path) - else: - logger.warning(f"❌ Failed to stop server on port {port}") - self.host.emit_error(f"Failed to stop execution on port {port}") - except Exception as e: - logger.error(f"❌ Error stopping server: {e}") - self.host.emit_error(f"Error stopping execution: {e}") - - thread = threading.Thread(target=kill_server, daemon=True) - thread.start() - - def disconnect(self) -> None: - """Disconnect ZMQ client (for cleanup).""" - if self.zmq_client is not None: - try: - self.zmq_client.disconnect() - except Exception as e: - logger.warning(f"Error disconnecting ZMQ client: {e}") - finally: - self.zmq_client = None diff --git a/openhcs/pyqt_gui/widgets/shared/time_travel_widget.py b/openhcs/pyqt_gui/widgets/shared/time_travel_widget.py index 1f2ec4e62..7c7e42243 100644 --- a/openhcs/pyqt_gui/widgets/shared/time_travel_widget.py +++ b/openhcs/pyqt_gui/widgets/shared/time_travel_widget.py @@ -9,8 +9,15 @@ from typing import Optional, List, Dict, Any from PyQt6.QtWidgets import ( - QWidget, QHBoxLayout, QSlider, QLabel, QPushButton, QToolTip, QFrame, QCheckBox, - QSizePolicy, QComboBox + QWidget, + QHBoxLayout, + QSlider, + QLabel, + QPushButton, + QToolTip, + QCheckBox, + QSizePolicy, + QComboBox, ) from PyQt6.QtCore import Qt, pyqtSignal, QPoint from PyQt6.QtGui import QFont @@ -35,10 +42,18 @@ class TimeTravelWidget(QWidget): - Tooltip showing num states captured """ + # OpenHCS-specific filter: hide system-only scopes from history + _HIDDEN_SCOPES = {"", "__plates__"} + # Emitted when time-travel position changes position_changed = pyqtSignal(int) # (index) - def __init__(self, color_scheme: Optional[ColorScheme] = None, show_browse_button: bool = True, parent=None): + def __init__( + self, + color_scheme: Optional[ColorScheme] = None, + show_browse_button: bool = True, + parent=None, + ): super().__init__(parent) self.color_scheme = color_scheme or ColorScheme() self.style_gen = StyleSheetGenerator(self.color_scheme) @@ -90,28 +105,26 @@ def _setup_ui(self): layout.addWidget(self.forward_btn) # Head button (return to present) - self.head_btn = self._create_icon_button("⏭", "Return to present (latest state)") + self.head_btn = self._create_icon_button( + "⏭", "Return to present (latest state)" + ) self.head_btn.clicked.connect(self._on_head) layout.addWidget(self.head_btn) - # Status label - self.status_label = QLabel("No history") - self.status_label.setMinimumWidth(180) - font = QFont() - font.setPointSize(9) - self.status_label.setFont(font) - layout.addWidget(self.status_label) - # Branch dropdown (auto-hides when only one branch) self.branch_combo = QComboBox() - self.branch_combo.setSizeAdjustPolicy(QComboBox.SizeAdjustPolicy.AdjustToContents) + self.branch_combo.setSizeAdjustPolicy( + QComboBox.SizeAdjustPolicy.AdjustToContents + ) self.branch_combo.setToolTip("Switch branch") self.branch_combo.currentTextChanged.connect(self._on_branch_changed) layout.addWidget(self.branch_combo) # Skip unsaved checkbox self.skip_unsaved_cb = QCheckBox("Saves only") - self.skip_unsaved_cb.setToolTip("Skip unsaved changes, jump only between save points") + self.skip_unsaved_cb.setToolTip( + "Skip unsaved changes, jump only between save points" + ) self.skip_unsaved_cb.setChecked(False) layout.addWidget(self.skip_unsaved_cb) @@ -121,11 +134,18 @@ def _setup_ui(self): self.browse_btn.clicked.connect(self._on_browse) layout.addWidget(self.browse_btn) - # Separator (only needed if browse button shown) - separator = QFrame() - separator.setFrameShape(QFrame.Shape.VLine) - separator.setFrameShadow(QFrame.Shadow.Sunken) - layout.addWidget(separator) + # Status label: right-most widget, text left-aligned + self.status_label = QLabel("No history") + self.status_label.setMinimumWidth(180) + self.status_label.setAlignment(Qt.AlignmentFlag.AlignLeft | Qt.AlignmentFlag.AlignVCenter) + self.status_label.setSizePolicy( + QSizePolicy.Policy.Expanding, + QSizePolicy.Policy.Preferred, + ) + font = QFont() + font.setPointSize(9) + self.status_label.setFont(font) + layout.addWidget(self.status_label) # Apply button styling self.setStyleSheet(self.style_gen.generate_button_style()) @@ -138,7 +158,9 @@ def _update_ui(self): History: [oldest, ..., newest] - index 0 = oldest, len-1 = head. Slider: 0 = oldest, max = head. Direct mapping, no inversion needed. """ - history = ObjectStateRegistry.get_history_info() + history = ObjectStateRegistry.get_history_info( + filter_fn=lambda scope_id: scope_id not in self._HIDDEN_SCOPES + ) if not history: self.slider.setMaximum(0) @@ -152,27 +174,31 @@ def _update_ui(self): self.slider.setEnabled(True) self.slider.setMaximum(len(history) - 1) - # Find current position - current_idx = next((h['index'] for h in history if h['is_current']), len(history) - 1) + # Find current position within the filtered list. + # NOTE: history entries include a global "index" which is not guaranteed to be + # contiguous after filtering; using it as a list index will crash. + current_pos = next( + (i for i, h in enumerate(history) if h["is_current"]), len(history) - 1 + ) self.slider.blockSignals(True) - self.slider.setValue(current_idx) + self.slider.setValue(current_pos) self.slider.blockSignals(False) # Update status - current = history[current_idx] + current = history[current_pos] is_traveling = ObjectStateRegistry.is_time_traveling() status = f"{current['timestamp']} | {current['label'][:30]}" if is_traveling: - status += f" ({current_idx + 1}/{len(history)})" + status += f" ({current_pos + 1}/{len(history)})" else: status += " (HEAD)" self.status_label.setText(status) # Enable/disable buttons (back = toward 0, forward = toward len-1) - self.back_btn.setEnabled(current_idx > 0) - self.forward_btn.setEnabled(current_idx < len(history) - 1) + self.back_btn.setEnabled(current_pos > 0) + self.forward_btn.setEnabled(current_pos < len(history) - 1) self.head_btn.setEnabled(is_traveling) # Visual indicator when time-traveling @@ -190,9 +216,9 @@ def _update_ui(self): self.branch_combo.blockSignals(True) self.branch_combo.clear() for b in branches: - self.branch_combo.addItem(b['name']) - if b['is_current']: - self.branch_combo.setCurrentText(b['name']) + self.branch_combo.addItem(b["name"]) + if b["is_current"]: + self.branch_combo.setCurrentText(b["name"]) self.branch_combo.blockSignals(False) def _on_branch_changed(self, name: str): @@ -202,7 +228,13 @@ def _on_branch_changed(self, name: str): def _on_slider_changed(self, value: int): """Handle slider value change. Direct mapping: slider value = history index.""" - ObjectStateRegistry.time_travel_to(value) + history = ObjectStateRegistry.get_history_info( + filter_fn=lambda scope_id: scope_id not in self._HIDDEN_SCOPES + ) + if value < 0 or value >= len(history): + return + + ObjectStateRegistry.time_travel_to(history[value]["index"]) self._update_ui() self.position_changed.emit(value) @@ -210,40 +242,47 @@ def _is_save_snapshot(self, label: str) -> bool: """Check if a snapshot label represents a save operation.""" return label.startswith("save") or label.startswith("init") - def _find_next_save_index(self, current_idx: int, direction: int) -> int: + def _find_next_save_index( + self, history: List[Dict[str, Any]], current_pos: int, direction: int + ) -> int: """Find next save snapshot in given direction. Args: - current_idx: Current history index (0 = oldest, len-1 = head) + history: Filtered history list. + current_pos: Current position in the filtered list (0 = oldest, len-1 = head) direction: -1 for back (toward older), +1 for forward (toward newer) Returns: Index of next save snapshot, or current if none found """ - history = ObjectStateRegistry.get_history_info() if not history: - return current_idx + return current_pos - search_idx = current_idx + direction - while 0 <= search_idx < len(history): - if self._is_save_snapshot(history[search_idx]['label']): - return search_idx - search_idx += direction + search_pos = current_pos + direction + while 0 <= search_pos < len(history): + if self._is_save_snapshot(history[search_pos]["label"]): + return search_pos + search_pos += direction # No save found - stay at current or go to boundary if direction > 0: return len(history) - 1 # Go to head - return current_idx + return current_pos def _on_back(self): """Step back in history (toward older = lower index).""" if self.skip_unsaved_cb.isChecked(): - history = ObjectStateRegistry.get_history_info() + history = ObjectStateRegistry.get_history_info( + filter_fn=lambda scope_id: scope_id not in self._HIDDEN_SCOPES + ) if history: - current_idx = next((h['index'] for h in history if h['is_current']), len(history) - 1) - target_idx = self._find_next_save_index(current_idx, -1) - if target_idx != current_idx: - ObjectStateRegistry.time_travel_to(target_idx) + current_pos = next( + (i for i, h in enumerate(history) if h["is_current"]), + len(history) - 1, + ) + target_pos = self._find_next_save_index(history, current_pos, -1) + if target_pos != current_pos: + ObjectStateRegistry.time_travel_to(history[target_pos]["index"]) else: ObjectStateRegistry.time_travel_back() self._update_ui() @@ -251,12 +290,17 @@ def _on_back(self): def _on_forward(self): """Step forward in history (toward newer = higher index).""" if self.skip_unsaved_cb.isChecked(): - history = ObjectStateRegistry.get_history_info() + history = ObjectStateRegistry.get_history_info( + filter_fn=lambda scope_id: scope_id not in self._HIDDEN_SCOPES + ) if history: - current_idx = next((h['index'] for h in history if h['is_current']), len(history) - 1) - target_idx = self._find_next_save_index(current_idx, +1) - if target_idx != current_idx: - ObjectStateRegistry.time_travel_to(target_idx) + current_pos = next( + (i for i, h in enumerate(history) if h["is_current"]), + len(history) - 1, + ) + target_pos = self._find_next_save_index(history, current_pos, +1) + if target_pos != current_pos: + ObjectStateRegistry.time_travel_to(history[target_pos]["index"]) else: ObjectStateRegistry.time_travel_forward() self._update_ui() @@ -268,19 +312,25 @@ def _on_head(self): def _on_browse(self): """Open the full Snapshot Browser window.""" - from openhcs.pyqt_gui.windows.snapshot_browser_window import SnapshotBrowserWindow + from openhcs.pyqt_gui.windows.snapshot_browser_window import ( + SnapshotBrowserWindow, + ) color_scheme = None parent = self.window() - if parent and hasattr(parent, 'color_scheme'): + if parent and hasattr(parent, "color_scheme"): color_scheme = parent.color_scheme - self._snapshot_browser = SnapshotBrowserWindow(color_scheme=color_scheme, parent=self) + self._snapshot_browser = SnapshotBrowserWindow( + color_scheme=color_scheme, parent=self + ) self._snapshot_browser.show() def _show_tooltip_at_position(self, value: int): """Show tooltip with snapshot details. Slider value = history index.""" - history = ObjectStateRegistry.get_history_info() + history = ObjectStateRegistry.get_history_info( + filter_fn=lambda scope_id: scope_id not in self._HIDDEN_SCOPES + ) if value < 0 or value >= len(history): return diff --git a/openhcs/pyqt_gui/widgets/shared/zmq_server_manager.py b/openhcs/pyqt_gui/widgets/shared/zmq_server_manager.py index c4beccd59..6395373e8 100644 --- a/openhcs/pyqt_gui/widgets/shared/zmq_server_manager.py +++ b/openhcs/pyqt_gui/widgets/shared/zmq_server_manager.py @@ -1,751 +1,581 @@ -""" -Generic ZMQ Server Manager Widget for PyQt6. +"""OpenHCS thin wrapper over generic ZMQ server browser widget.""" -Provides a reusable UI component for managing any ZMQ server (execution servers, -Napari viewers, future servers) using the ZMQServer/ZMQClient ABC interface. - -Features: -- Auto-discovery of running servers via port scanning -- Display server info (port, type, status, log file) -- Graceful shutdown and force kill -- Double-click to open log files -- Works with ANY ZMQServer subclass -- Tracks launching viewers with queued image counts -""" +from __future__ import annotations import logging from pathlib import Path -from typing import List, Dict, Any, Optional -from PyQt6.QtWidgets import ( - QWidget, QVBoxLayout, QHBoxLayout, QTreeWidget, QTreeWidgetItem, - QPushButton, QGroupBox, QMessageBox, QAbstractItemView +from typing import Any, Dict, List, Optional + +from PyQt6.QtCore import QTimer, Qt, pyqtSlot +from PyQt6.QtWidgets import QTreeWidgetItem + +from pyqt_reactive.services import ( + BaseServerInfo, + DefaultServerInfoParser, + ExecutionServerInfo, + ServerInfoParserABC, + ZMQServerScanService, ) -from PyQt6.QtCore import Qt, pyqtSignal, pyqtSlot, QTimer from pyqt_reactive.theming import StyleSheetGenerator -import threading +from pyqt_reactive.widgets.shared import ( + KillOperationPlan, + TreeSyncAdapter, + ZMQServerBrowserWidgetABC, +) +from zmqruntime.viewer_state import ViewerStateManager + +from openhcs.core.progress import ProgressEvent, registry +from openhcs.pyqt_gui.widgets.shared.server_browser import ( + ProgressNode, + ProgressTopologyState, + ProgressTreeBuilder, + ServerKillService, + ServerRowPresenter, + summarize_execution_server, +) logger = logging.getLogger(__name__) -# Global registry for launching viewers -# Format: {port: {'type': 'napari'|'fiji', 'queued_images': int, 'start_time': float}} -_launching_viewers_lock = threading.Lock() -_launching_viewers: Dict[int, Dict[str, Any]] = {} - - -# Global reference to active ZMQ server manager widgets (for triggering refreshes) -_active_managers_lock = threading.Lock() -_active_managers: List['ZMQServerManagerWidget'] = [] - - -def register_launching_viewer(port: int, viewer_type: str, queued_images: int = 0): - """Register a viewer that is launching and trigger UI refresh. - - If the viewer is already launching, accumulates the queue count instead of replacing it. - """ - import time - with _launching_viewers_lock: - if port in _launching_viewers: - # Already launching - accumulate queue count - _launching_viewers[port]['queued_images'] += queued_images - logger.info(f"Updated launching {viewer_type} viewer on port {port}: added {queued_images} images (total: {_launching_viewers[port]['queued_images']})") - else: - # New launching viewer - _launching_viewers[port] = { - 'type': viewer_type, - 'queued_images': queued_images, - 'start_time': time.time() - } - logger.info(f"Registered launching {viewer_type} viewer on port {port} with {queued_images} queued images") - - # Trigger immediate refresh on all active managers (fast path - no port scan) - _trigger_manager_refresh_fast() - - -def update_launching_viewer_queue(port: int, queued_images: int): - """Update the queued image count for a launching viewer and trigger UI refresh. - - This SETS the queue count (doesn't accumulate). Use register_launching_viewer() to add images. - """ - with _launching_viewers_lock: - if port in _launching_viewers: - _launching_viewers[port]['queued_images'] = queued_images - logger.debug(f"Updated launching viewer on port {port}: {queued_images} queued images") - - # Trigger immediate refresh on all active managers (fast path - no port scan) - _trigger_manager_refresh_fast() - - -def unregister_launching_viewer(port: int): - """Remove a viewer from the launching registry (it's now ready) and trigger UI refresh.""" - with _launching_viewers_lock: - if port in _launching_viewers: - del _launching_viewers[port] - logger.info(f"Unregistered launching viewer on port {port} (now ready)") - - # Trigger full refresh to pick up the now-ready viewer via port scan - _trigger_manager_refresh_full() - - -def _trigger_manager_refresh_fast(): - """Trigger fast refresh (launching viewers only, no port scan) on all active managers.""" - with _active_managers_lock: - for manager in _active_managers: - try: - # Use QMetaObject to safely call from any thread - from PyQt6.QtCore import QMetaObject, Qt - QMetaObject.invokeMethod( - manager, - "_refresh_launching_viewers_only", - Qt.ConnectionType.QueuedConnection - ) - except Exception as e: - logger.debug(f"Failed to trigger fast refresh on manager: {e}") - - -def _trigger_manager_refresh_full(): - """Trigger full refresh (port scan + launching viewers) on all active managers.""" - with _active_managers_lock: - for manager in _active_managers: - try: - # Use QMetaObject to safely call from any thread - from PyQt6.QtCore import QMetaObject, Qt - QMetaObject.invokeMethod( - manager, - "refresh_servers", - Qt.ConnectionType.QueuedConnection - ) - except Exception as e: - logger.debug(f"Failed to trigger full refresh on manager: {e}") - - -def get_launching_viewers() -> Dict[int, Dict[str, Any]]: - """Get a copy of the launching viewers registry.""" - with _launching_viewers_lock: - return dict(_launching_viewers) - - -class ZMQServerManagerWidget(QWidget): - """ - Generic ZMQ server manager widget. - - Works with any ZMQServer subclass via the ABC interface. - Displays running servers and provides management controls. - """ - - # Signals - server_killed = pyqtSignal(int) # Emitted when server is killed (port) - log_file_opened = pyqtSignal(str) # Emitted when log file is opened (path) - _scan_complete = pyqtSignal(list) # Internal signal for async scan completion - _kill_complete = pyqtSignal(bool, str) # Internal signal for async kill completion (success, message) +class ZMQServerManagerWidget(ZMQServerBrowserWidgetABC): + """OpenHCS adapter for generic ZMQ browser UI + OpenHCS progress semantics.""" def __init__( self, ports_to_scan: List[int], title: str = "ZMQ Servers", style_generator: Optional[StyleSheetGenerator] = None, - parent: Optional[QWidget] = None + server_info_parser: Optional[ServerInfoParserABC] = None, + parent=None, ): - """ - Initialize ZMQ server manager widget. - - Args: - ports_to_scan: List of ports to scan for servers - title: Title for the group box - style_generator: Style generator for consistent styling - parent: Parent widget - """ - super().__init__(parent) + if style_generator is None: + raise RuntimeError("style_generator is required for ZMQServerManagerWidget") - self.ports_to_scan = ports_to_scan - self.title = title - self.style_generator = style_generator - - # Server tracking - self.servers: List[Dict[str, Any]] = [] + from openhcs.constants.constants import CONTROL_PORT_OFFSET + from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG - # Register this manager for launching viewer updates - with _active_managers_lock: - _active_managers.append(self) + parser = server_info_parser or DefaultServerInfoParser() + scan_service = ZMQServerScanService( + control_port_offset=CONTROL_PORT_OFFSET, + config=OPENHCS_ZMQ_CONFIG, + host="localhost", + ) + super().__init__( + ports_to_scan=ports_to_scan, + title=title, + style_generator=style_generator, + scan_service=scan_service, + server_info_parser=parser, + parent=parent, + ) - # Connect internal signal for async scanning - self._scan_complete.connect(self._update_server_list) + def _manager_callback(_instance) -> None: + try: + from PyQt6.QtCore import QMetaObject, Qt - # Auto-refresh timer (async scanning won't block UI) - self.refresh_timer = QTimer() - self.refresh_timer.timeout.connect(self.refresh_servers) + QMetaObject.invokeMethod( + self, + "_refresh_launching_viewers_only", + Qt.ConnectionType.QueuedConnection, + ) + except Exception as error: + logger.debug("Viewer state callback invocation failed: %s", error) + + mgr = ViewerStateManager.get_instance() + mgr.register_state_callback(_manager_callback) + self._viewer_state_callback = _manager_callback + self._viewer_state_callback_registered = True + + self._progress_tracker = registry() + self._registry_listener = self._on_registry_event + self._progress_tracker.add_listener(self._registry_listener) + self._registry_listener_registered = True + self._progress_dirty = False + self._topology_state = ProgressTopologyState() + self._known_wells = self._topology_state.known_wells + self._worker_assignments = self._topology_state.worker_assignments + self._seen_execution_ids = self._topology_state.seen_execution_ids + + self._zmq_client = None + + self._tree_sync_adapter = TreeSyncAdapter() + + self._progress_tree_builder = ProgressTreeBuilder() + self._server_kill_service = ServerKillService() + self._server_row_presenter = ServerRowPresenter( + create_tree_item=self._create_tree_item, + update_execution_server_item=self._update_execution_server_item, + log_warning=logger.warning, + ) + self._missing_port_counts: Dict[int, int] = {} + + # Real-time progress timer for smooth UI updates during execution + self._progress_timer = QTimer() + self._progress_timer.timeout.connect(self._update_from_progress) + self._progress_timer.start(100) # 100ms for smooth updates + + def _populate_tree(self, parsed_servers: List[BaseServerInfo]) -> None: + """Populate tree with servers, avoiding duplicates since tree.clear() is bypassed.""" + scanned_ports = {info.port for info in parsed_servers} + for port in scanned_ports: + self._missing_port_counts.pop(port, None) + + # Add launching viewers (being auto-started from streaming) + self._sync_launching_viewers(scanned_ports) + + # Add/update servers from scan + for server_info in parsed_servers: + self._sync_server_item(server_info) + + # Remove servers only after consecutive misses to avoid transient flicker. + for idx in range(self.server_tree.topLevelItemCount() - 1, -1, -1): + item = self.server_tree.topLevelItem(idx) + if item is None: + continue + data = item.data(0, Qt.ItemDataRole.UserRole) + if not isinstance(data, dict): + continue + port = data.get("port") + if port is None: + continue + if port in scanned_ports: + continue - # Cleanup flag to prevent operations after cleanup - self._is_cleaning_up = False + # Check if this is a launching viewer (being auto-started from streaming) + if data.get("launching"): + # Don't count launching viewers as missing - they persist until ready + self._missing_port_counts.pop(port, None) + continue - self.setup_ui() + # Check if server has active executions before removing + last_known = self._last_known_servers.get(port, {}) + running_execs = last_known.get("running_executions", []) + active_execution_ids = [ + str(exec_info.get("execution_id")) + for exec_info in running_execs + if exec_info.get("execution_id") + ] + tracker_exec_ids = set(self._progress_tracker.get_execution_ids()) + has_active_executions = any( + exec_id in tracker_exec_ids for exec_id in active_execution_ids + ) - def cleanup(self): - """Cleanup resources before widget destruction.""" - if self._is_cleaning_up: - return + if has_active_executions: + # Server is busy with active executions, don't remove it + self._missing_port_counts.pop(port, None) + continue - self._is_cleaning_up = True - - # Stop refresh timer first to prevent new refresh calls - if hasattr(self, 'refresh_timer') and self.refresh_timer: - self.refresh_timer.stop() - self.refresh_timer.deleteLater() - self.refresh_timer = None - - # Unregister this manager from global list - with _active_managers_lock: - if self in _active_managers: - _active_managers.remove(self) - - logger.debug("ZMQServerManagerWidget cleanup completed") - - def __del__(self): - """Cleanup when widget is destroyed.""" - self.cleanup() - - def showEvent(self, event): - """Auto-scan for servers when widget is shown.""" - super().showEvent(event) - if not self._is_cleaning_up: - # Scan for servers on first show - self.refresh_servers() - # Start auto-refresh (1 second interval - async scanning won't block UI) - if self.refresh_timer: - self.refresh_timer.start(1000) - - def hideEvent(self, event): - """Stop auto-refresh when widget is hidden.""" - super().hideEvent(event) - # Stop timer to prevent unnecessary background work - if hasattr(self, 'refresh_timer') and self.refresh_timer: - self.refresh_timer.stop() - - def setup_ui(self): - """Setup the user interface.""" - layout = QVBoxLayout(self) - layout.setContentsMargins(0, 0, 0, 0) - - # Group box - group_box = QGroupBox(self.title) - group_layout = QVBoxLayout(group_box) - group_layout.setContentsMargins(5, 5, 5, 5) - - # Server tree (hierarchical display with workers as children) - self.server_tree = QTreeWidget() - self.server_tree.setHeaderLabels(["Server / Worker", "Status", "Info"]) - self.server_tree.setSelectionMode(QAbstractItemView.SelectionMode.ExtendedSelection) - self.server_tree.itemDoubleClicked.connect(self._on_item_double_clicked) - self.server_tree.setColumnWidth(0, 250) - self.server_tree.setColumnWidth(1, 100) - group_layout.addWidget(self.server_tree) - - # Buttons - button_layout = QHBoxLayout() - - self.refresh_btn = QPushButton("Refresh") - self.refresh_btn.clicked.connect(self.refresh_servers) - button_layout.addWidget(self.refresh_btn) - - self.quit_btn = QPushButton("Quit") - self.quit_btn.clicked.connect(self.quit_selected_servers) - button_layout.addWidget(self.quit_btn) - - self.force_kill_btn = QPushButton("Force Kill") - self.force_kill_btn.clicked.connect(self.force_kill_selected_servers) - button_layout.addWidget(self.force_kill_btn) - - group_layout.addLayout(button_layout) - - layout.addWidget(group_box) - - # Apply styling - if self.style_generator: - # Apply button styles - self.refresh_btn.setStyleSheet(self.style_generator.generate_button_style()) - self.quit_btn.setStyleSheet(self.style_generator.generate_button_style()) - self.force_kill_btn.setStyleSheet(self.style_generator.generate_button_style()) - - # Apply tree widget style (uses existing method) - self.server_tree.setStyleSheet(self.style_generator.generate_tree_widget_style()) - - # Apply group box style - cs = self.style_generator.color_scheme - group_box.setStyleSheet(f""" - QGroupBox {{ - background-color: {cs.to_hex(cs.panel_bg)}; - border: 1px solid {cs.to_hex(cs.border_color)}; - border-radius: 4px; - margin-top: 8px; - padding-top: 8px; - font-weight: bold; - color: {cs.to_hex(cs.text_accent)}; - }} - QGroupBox::title {{ - subcontrol-origin: margin; - subcontrol-position: top left; - padding: 2px 5px; - color: {cs.to_hex(cs.text_accent)}; - }} - """) - - # Connect internal signals - self._scan_complete.connect(self._update_server_list) - self._kill_complete.connect(self._on_kill_complete) - - def refresh_servers(self): - """Scan ports and refresh server list (async in background).""" - # Guard against calls after cleanup - if self._is_cleaning_up: - return + misses = self._missing_port_counts.get(port, 0) + 1 + self._missing_port_counts[port] = misses + if misses < 2: + continue - import threading - - def scan_and_update(): - """Background thread to scan ports without blocking UI.""" - import concurrent.futures - - # Scan ports in parallel using thread pool (like Napari implementation) - servers = [] - - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - # Submit all ping tasks - future_to_port = { - executor.submit(self._ping_server, port): port - for port in self.ports_to_scan - } - - # Collect results as they complete - for future in concurrent.futures.as_completed(future_to_port): - port = future_to_port[future] - try: - server_info = future.result() - if server_info: - servers.append(server_info) - except Exception as e: - logger.debug(f"Error scanning port {port}: {e}") - - # Update UI on main thread via signal - self._scan_complete.emit(servers) - - # Start scan in background thread - thread = threading.Thread(target=scan_and_update, daemon=True) - thread.start() - - def _ping_server(self, port: int) -> Optional[Dict[str, Any]]: - """ - Ping a server on the given port and return its info. - - Returns server info dict if responsive, None otherwise. - """ - import zmq - import pickle - from openhcs.constants.constants import CONTROL_PORT_OFFSET - from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG - from zmqruntime.transport import get_zmq_transport_url, get_default_transport_mode + self._missing_port_counts.pop(port, None) + self.server_tree.takeTopLevelItem(idx) - control_port = port + CONTROL_PORT_OFFSET - control_context = None - control_socket = None + def _find_existing_server_item(self, port: int) -> Optional[QTreeWidgetItem]: + """Find existing server item by port.""" + for idx in range(self.server_tree.topLevelItemCount()): + item = self.server_tree.topLevelItem(idx) + data = item.data(0, Qt.ItemDataRole.UserRole) + if isinstance(data, dict) and data.get("port") == port: + return item + return None + + def _sync_server_item(self, server_info: BaseServerInfo) -> None: + """Sync a server item - update existing or create new.""" + existing_item = self._find_existing_server_item(server_info.port) + status_icon = "✅" if server_info.ready else "🚀" + rendered_item = self._server_row_presenter.render_server( + server_info, status_icon + ) - try: - control_context = zmq.Context() - control_socket = control_context.socket(zmq.REQ) - control_socket.setsockopt(zmq.LINGER, 0) - control_socket.setsockopt(zmq.RCVTIMEO, 300) # 300ms timeout for fast scanning - - # Use transport mode-aware URL (IPC or TCP) - transport_mode = get_default_transport_mode() - control_url = get_zmq_transport_url( - control_port, - host="localhost", - mode=transport_mode, - config=OPENHCS_ZMQ_CONFIG, + if existing_item is not None: + if rendered_item is not None: + existing_item.setText(0, rendered_item.text(0)) + if not isinstance(server_info, ExecutionServerInfo): + existing_item.setText(1, rendered_item.text(1)) + existing_item.setText(2, rendered_item.text(2)) + existing_item.setData(0, Qt.ItemDataRole.UserRole, server_info.raw) + self._server_row_presenter.populate_server_children( + server_info, existing_item ) - control_socket.connect(control_url) + return - # Send ping - ping_message = {'type': 'ping'} - control_socket.send(pickle.dumps(ping_message)) + if rendered_item is None: + return - # Wait for pong - response = control_socket.recv() - response_data = pickle.loads(response) + rendered_item.setData(0, Qt.ItemDataRole.UserRole, server_info.raw) + self.server_tree.addTopLevelItem(rendered_item) + self._server_row_presenter.populate_server_children(server_info, rendered_item) - # Return server info if valid pong - if response_data.get('type') == 'pong': - return response_data + def _sync_launching_viewers(self, scanned_ports: set[int]) -> None: + """Add launching viewer entries to tree for viewers being auto-started.""" + from zmqruntime.viewer_state import ViewerStateManager, ViewerState - return None + mgr = ViewerStateManager.get_instance() + launching_viewers = { + viewer.port: viewer + for viewer in mgr.list_viewers() + if viewer.state == ViewerState.LAUNCHING + } - except Exception: - return None - finally: - if control_socket: - try: - control_socket.close() - except: - pass - if control_context: - try: - control_context.term() - except: - pass - - @pyqtSlot() - def _refresh_launching_viewers_only(self): - """Fast refresh: Update UI with launching viewers only (no port scan). + for port, viewer in launching_viewers.items(): + # Skip if already scanned (viewer is now running) + if port in scanned_ports: + continue - This is called when launching viewer state changes and provides instant feedback. - """ - # Guard against calls after cleanup - if self._is_cleaning_up: - return + # Check if already in tree + existing_item = self._find_existing_server_item(port) + if existing_item is not None: + # Update existing item + viewer_type = viewer.viewer_type.capitalize() + queued = viewer.queued_images + info_text = f"{queued} images queued" if queued > 0 else "Starting..." + existing_item.setText(0, f"Port {port} - {viewer_type} Viewer") + existing_item.setText(1, "🚀 Launching") + existing_item.setText(2, info_text) + continue - # Keep existing scanned servers, just update the tree display - self._update_server_list(self.servers) + # Create new tree item for launching viewer + viewer_type = viewer.viewer_type.capitalize() + queued = viewer.queued_images + info_text = f"{queued} images queued" if queued > 0 else "Starting..." + + item = QTreeWidgetItem() + item.setText(0, f"Port {port} - {viewer_type} Viewer") + item.setText(1, "🚀 Launching") + item.setText(2, info_text) + item.setData( + 0, + Qt.ItemDataRole.UserRole, + {"port": port, "launching": True, "viewer_type": viewer.viewer_type}, + ) + self.server_tree.addTopLevelItem(item) @pyqtSlot(list) - def _update_server_list(self, servers: List[Dict[str, Any]]): - """Update server tree on UI thread (called via signal).""" - from zmqruntime.queue_tracker import GlobalQueueTrackerRegistry - + def _update_server_list(self, servers: List[Dict[str, Any]]) -> None: + """Override to bypass TreeRebuildCoordinator's tree.clear() which causes flicker.""" self.servers = servers + parsed_servers = [self._server_info_parser.parse(server) for server in servers] - # Save current selection (by port) before clearing - selected_ports = set() - for item in self.server_tree.selectedItems(): - server_data = item.data(0, Qt.ItemDataRole.UserRole) - if server_data and 'port' in server_data: - selected_ports.add(server_data['port']) + for server in servers: + port = server.get("port") + if port: + self._last_known_servers[port] = server - self.server_tree.clear() + # Direct call to populate_tree bypasses the rebuild coordinator + self._populate_tree(parsed_servers) - # Get queue tracker registry for progress info - registry = GlobalQueueTrackerRegistry() + def _periodic_domain_cleanup(self) -> None: + removed = self._progress_tracker.cleanup_old_executions() + if removed > 0: + logger.info(f"Periodic cleanup: removed {removed} old completed executions") - # First, add launching viewers - launching_viewers = get_launching_viewers() - for port, info in launching_viewers.items(): - viewer_type = info['type'].capitalize() - queued_images = info['queued_images'] + def _kill_ports_with_plan( + self, + *, + ports: List[int], + plan: KillOperationPlan, + on_server_killed, + ) -> tuple[bool, str]: + return self._server_kill_service.kill_ports( + ports=ports, + plan=plan, + on_server_killed=on_server_killed, + log_info=logger.info, + log_warning=logger.warning, + log_error=logger.error, + ) - display_text = f"Port {port} - {viewer_type} Viewer" - status_text = "🚀 Launching" - info_text = f"{queued_images} images queued" if queued_images > 0 else "Starting..." + def _on_browser_shown(self) -> None: + self._setup_progress_client() - item = QTreeWidgetItem([display_text, status_text, info_text]) - item.setData(0, Qt.ItemDataRole.UserRole, {'port': port, 'launching': True}) - self.server_tree.addTopLevelItem(item) + def _on_browser_hidden(self) -> None: + if self._zmq_client is not None: + self._zmq_client.disconnect() + self._zmq_client = None - # Add servers that are processing images (even if they didn't respond to ping) - # This prevents busy servers from disappearing during image processing - scanned_ports = {server.get('port') for server in servers} - for tracker_port, tracker in registry.get_all_trackers().items(): - if tracker_port in scanned_ports or tracker_port in launching_viewers: - continue # Already in the list - - # Check if this tracker has pending images (server is busy processing) - pending = tracker.get_pending_count() - if pending > 0: - # Server is busy processing - add it even though it didn't respond to ping - processed, total = tracker.get_progress() - viewer_type = tracker.viewer_type.capitalize() - - display_text = f"Port {tracker_port} - {viewer_type}ViewerServer" - status_text = "⚙️" # Busy icon - info_text = f"Processing: {processed}/{total} images" - - # Check for stuck images - if tracker.has_stuck_images(): - status_text = "⚠️" - stuck_images = tracker.get_stuck_images() - info_text += f" (⚠️ {len(stuck_images)} stuck)" - - # Create pseudo-server dict for consistency - pseudo_server = { - 'port': tracker_port, - 'server': f'{viewer_type}ViewerServer', - 'ready': True, - 'busy': True # Mark as busy - } - - item = QTreeWidgetItem([display_text, status_text, info_text]) - item.setData(0, Qt.ItemDataRole.UserRole, pseudo_server) - self.server_tree.addTopLevelItem(item) - - # Then add running servers - for server in servers: - port = server.get('port', 'unknown') + def _on_browser_cleanup(self) -> None: + if self._zmq_client is not None: + try: + self._zmq_client.disconnect() + except Exception as error: + logger.warning( + "Failed to disconnect ZMQ client during cleanup: %s", error + ) + self._zmq_client = None + + if self._viewer_state_callback_registered: + mgr = ViewerStateManager.get_instance() + if self._viewer_state_callback: + mgr.unregister_state_callback(self._viewer_state_callback) + self._viewer_state_callback_registered = False + + if self._registry_listener_registered: + removed = self._progress_tracker.remove_listener(self._registry_listener) + if not removed: + raise RuntimeError( + "ZMQServerManagerWidget listener removal failed: listener not registered" + ) + self._registry_listener_registered = False - # Skip if this port is in launching registry (shouldn't happen, but be safe) - if port in launching_viewers: - continue + for execution_id in list(self._seen_execution_ids): + self._progress_tracker.clear_execution(execution_id) + self._topology_state.clear_execution(execution_id) + self._topology_state.clear_all() - server_type = server.get('server', 'Unknown') - ready = server.get('ready', False) - - # Determine status icon - if ready: - status_icon = "✅" - else: - status_icon = "🚀" - - # Handle execution servers specially - show workers as children - if server_type == 'ZMQExecutionServer': - running_executions = server.get('running_executions', []) - workers = server.get('workers', []) - - # Create server item - if running_executions: - server_text = f"Port {port} - Execution Server" - status_text = f"{status_icon} {len(running_executions)} exec" - info_text = f"{len(workers)} workers" - else: - server_text = f"Port {port} - Execution Server" - status_text = f"{status_icon} Idle" - info_text = f"{len(workers)} workers" if workers else "" - - server_item = QTreeWidgetItem([server_text, status_text, info_text]) - server_item.setData(0, Qt.ItemDataRole.UserRole, server) - self.server_tree.addTopLevelItem(server_item) - - # Add worker processes as children - for worker in workers: - pid = worker.get('pid', 'unknown') - status = worker.get('status', 'unknown') - cpu = worker.get('cpu_percent', 0) - mem_mb = worker.get('memory_mb', 0) - - worker_text = f" Worker PID {pid}" - worker_status = f"⚙️ {status}" - worker_info = f"CPU: {cpu:.1f}% | Mem: {mem_mb:.0f}MB" - - worker_item = QTreeWidgetItem([worker_text, worker_status, worker_info]) - worker_item.setData(0, Qt.ItemDataRole.UserRole, {'type': 'worker', 'pid': pid, 'server': server}) - server_item.addChild(worker_item) - - # Expand server item if it has workers - if workers: - server_item.setExpanded(True) - - else: - # Other server types (Napari, Fiji viewers) - show with progress if available - display_text = f"Port {port} - {server_type}" - status_text = status_icon - info_text = "" - - # Check if this is a viewer with pending images - tracker = registry.get_tracker(port) - if tracker: - processed, total = tracker.get_progress() - pending = tracker.get_pending_count() - - if pending > 0: - # Still processing images - info_text = f"Processing: {processed}/{total} images" - - # Check for stuck images - if tracker.has_stuck_images(): - status_text = "⚠️" # Warning icon for stuck - stuck_images = tracker.get_stuck_images() - info_text += f" (⚠️ {len(stuck_images)} stuck)" - elif total > 0: - # All images processed - info_text = f"✅ Processed {total} images" - - # If no processing info, show memory usage - if not info_text: - mem_mb = server.get('memory_mb') - cpu_percent = server.get('cpu_percent') - if mem_mb is not None: - info_text = f"Mem: {mem_mb:.0f}MB" - if cpu_percent is not None: - info_text += f" | CPU: {cpu_percent:.1f}%" - - item = QTreeWidgetItem([display_text, status_text, info_text]) - item.setData(0, Qt.ItemDataRole.UserRole, server) - self.server_tree.addTopLevelItem(item) - - # Restore selection after refresh - if selected_ports: - for i in range(self.server_tree.topLevelItemCount()): - item = self.server_tree.topLevelItem(i) - server_data = item.data(0, Qt.ItemDataRole.UserRole) - if server_data and server_data.get('port') in selected_ports: - item.setSelected(True) - - logger.debug(f"Found {len(servers)} ZMQ servers") - - @pyqtSlot(bool, str) - def _on_kill_complete(self, success: bool, message: str): - """Handle kill operation completion on UI thread.""" - if not success: - QMessageBox.warning(self, "Kill Failed", message) - # Refresh list after kill (quick refresh for better UX) - QTimer.singleShot(200, self.refresh_servers) - - def quit_selected_servers(self): - """Gracefully quit selected servers (async to avoid blocking UI).""" - selected_items = self.server_tree.selectedItems() - if not selected_items: - QMessageBox.warning(self, "No Selection", "Please select servers to quit.") - return + if self._progress_timer is not None: + self._progress_timer.stop() + self._progress_timer.deleteLater() + self._progress_timer = None - # Collect ports to kill BEFORE showing dialog (items may be deleted by auto-refresh) - ports_to_kill = [] - for item in selected_items: - data = item.data(0, Qt.ItemDataRole.UserRole) - # Skip worker items (they don't have ports) - if data and data.get('type') == 'worker': - continue - port = data.get('port') if data else None - if port: - ports_to_kill.append(port) + def _setup_progress_client(self) -> None: + from openhcs.runtime.zmq_execution_client import ZMQExecutionClient - if not ports_to_kill: - QMessageBox.warning(self, "No Servers", "No servers selected (only workers selected).") - return + if self._zmq_client is not None: + try: + self._zmq_client.disconnect() + except Exception as error: + logger.warning("Failed to disconnect existing ZMQ client: %s", error) + self._zmq_client = None - # Confirm with user - reply = QMessageBox.question( - self, - "Quit Confirmation", - f"Gracefully quit {len(ports_to_kill)} server(s)?\n\n" - "For execution servers: kills workers only, server stays alive.", - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - QMessageBox.StandardButton.Yes + try: + logger.debug("_setup_progress_client: creating new ZMQExecutionClient") + self._zmq_client = ZMQExecutionClient( + port=7777, + persistent=True, + progress_callback=self._on_progress, + ) + connected = self._zmq_client.connect(timeout=1) + if not connected: + logger.warning("_setup_progress_client: failed to connect") + self._zmq_client = None + return + logger.debug( + "_setup_progress_client: connected, starting progress listener" + ) + self._zmq_client._start_progress_listener() + except Exception as error: + logger.warning("Failed to connect to execution server: %s", error) + self._zmq_client = None + + def _on_progress(self, message: dict) -> None: + event = ProgressEvent.from_dict(message) + logger.debug( + f"_on_progress: exec={event.execution_id[:8] if event.execution_id else None}, phase={event.phase}, status={event.status}" + ) + self._topology_state.register_event(event) + self._progress_tracker.register_event(event.execution_id, event) + logger.debug( + f"_on_progress: tracker now has {len(self._progress_tracker.get_execution_ids())} executions" ) - if reply != QMessageBox.StandardButton.Yes: + def _on_registry_event(self, _execution_id: str, _event: ProgressEvent) -> None: + """Mark progress dirty when registry changes - triggers timer update.""" + self._progress_dirty = True + + @pyqtSlot() + def _update_from_progress(self) -> None: + """Real-time progress update - called every 100ms by timer.""" + dirty = getattr(self, "_progress_dirty", False) + if not dirty: return + self._progress_dirty = False + try: + for index in range(self.server_tree.topLevelItemCount()): + item = self.server_tree.topLevelItem(index) + if item is None: + continue + data = item.data(0, Qt.ItemDataRole.UserRole) + if not isinstance(data, dict): + continue + try: + parsed_server_info = self._server_info_parser.parse(data) + except Exception: + continue + if isinstance(parsed_server_info, ExecutionServerInfo): + self._update_execution_server_item(item, data) + except Exception as error: + logger.exception("Error updating from progress: %s", error) + + def _get_plate_name(self, plate_id: str, exec_id: str | None = None) -> str: + plate_leaf = Path(plate_id).name + if exec_id: + return f"{plate_leaf} ({exec_id[:8]})" + return plate_leaf + + def _build_progress_tree(self, executions: Dict[str, list]) -> List[ProgressNode]: + return self._progress_tree_builder.build_progress_tree( + executions=executions, + worker_assignments=self._worker_assignments, + known_wells=self._known_wells, + step_names=self._topology_state.step_names, + get_plate_name=self._get_plate_name, + ) - # Kill in background thread to avoid blocking UI - import threading + def _update_execution_server_item( + self, server_item: QTreeWidgetItem, server_data: dict + ) -> None: + try: + executions = { + execution_id: self._progress_tracker.get_events(execution_id) + for execution_id in self._progress_tracker.get_execution_ids() + } + logger.debug( + f"_update_exec_server: tracker has {len(executions)} executions, progress events: {list(executions.keys())}" + ) + for exec_id, events in executions.items(): + logger.debug( + f" exec={exec_id[:8] if exec_id else None}, events={len(events) if events else 0}" + ) - def kill_servers(): - from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG - from zmqruntime.client import ZMQClient - from zmqruntime.queue_tracker import GlobalQueueTrackerRegistry - failed_ports = [] - registry = GlobalQueueTrackerRegistry() + parsed_server_info = self._server_info_parser.parse(server_data) + if not isinstance(parsed_server_info, ExecutionServerInfo): + raise ValueError( + "Expected ExecutionServerInfo for execution subtree update, " + f"got {type(parsed_server_info).__name__}" + ) - for port in ports_to_kill: - try: - logger.info(f"Attempting to quit server on port {port}...") - success = ZMQClient.kill_server_on_port(port, graceful=True, config=OPENHCS_ZMQ_CONFIG) - if success: - logger.info(f"✅ Successfully quit server on port {port}") - # Clear queue tracker for this viewer - registry.remove_tracker(port) - self.server_killed.emit(port) - else: - failed_ports.append(port) - logger.warning(f"❌ Failed to quit server on port {port} (kill_server_on_port returned False)") - except Exception as e: - failed_ports.append(port) - logger.error(f"❌ Error quitting server on port {port}: {e}") - - # Emit completion signal - if failed_ports: - self._kill_complete.emit(False, f"Failed to quit servers on ports: {failed_ports}") - else: - self._kill_complete.emit(True, "All servers quit successfully") - - thread = threading.Thread(target=kill_servers, daemon=True) - thread.start() - - def force_kill_selected_servers(self): - """Force kill selected servers (async to avoid blocking UI).""" - selected_items = self.server_tree.selectedItems() - if not selected_items: - QMessageBox.warning(self, "No Selection", "Please select servers to force kill.") - return + nodes = self._build_progress_tree(executions) if executions else [] + for node in nodes: + logger.debug( + f" plate={node.node_id.split('/')[-1] if node.node_id else None}, status={node.status}, percent={node.percent}" + ) + nodes = self._merge_server_snapshot_nodes(nodes, parsed_server_info) + for node in nodes: + logger.debug( + f" MERGED plate={node.node_id.split('/')[-1] if node.node_id else None}, status={node.status}, percent={node.percent}" + ) - # Collect ports to kill BEFORE showing dialog (items may be deleted by auto-refresh) - ports_to_kill = [] - for item in selected_items: - data = item.data(0, Qt.ItemDataRole.UserRole) - # Skip worker items (they don't have ports) - if data and data.get('type') == 'worker': + summary = summarize_execution_server(nodes) + logger.debug( + f" SUMMARY: status={summary.status_text}, info={summary.info_text}" + ) + server_item.setText(1, summary.status_text) + server_item.setText(2, summary.info_text) + self._tree_sync_adapter.sync_children( + server_item, + self._progress_tree_builder.to_tree_nodes(nodes), + ) + except Exception as error: + logger.exception("Error updating execution server item: %s", error) + + def _merge_server_snapshot_nodes( + self, nodes: List[ProgressNode], server_info: ExecutionServerInfo + ) -> List[ProgressNode]: + by_plate_id: Dict[str, ProgressNode] = {node.node_id: node for node in nodes} + running_execution_ids = { + running.execution_id for running in server_info.running_execution_entries + } + running_plate_ids = { + running.plate_id for running in server_info.running_execution_entries + } + + for running in server_info.running_execution_entries: + plate_id = running.plate_id + execution_id = running.execution_id + running_status = "⏳ Compiling" if running.compile_only else "⚙️ Executing" + existing = by_plate_id.get(plate_id) + + if existing is None: + plate_name = self._get_plate_name(plate_id, execution_id) + node = ProgressNode( + node_id=plate_id, + node_type="plate", + label=f"📋 {plate_name}", + status=running_status, + info="0.0%", + execution_id=execution_id, + percent=0.0, + children=[], + ) + nodes.append(node) + by_plate_id[plate_id] = node continue - port = data.get('port') if data else None - if port: - ports_to_kill.append(port) - if not ports_to_kill: - QMessageBox.warning(self, "No Servers", "No servers selected (only workers selected).") - return - - # Confirm with user - reply = QMessageBox.question( - self, - "Force Kill Confirmation", - f"Force kill {len(ports_to_kill)} server(s)?\n\n" - "For execution servers: kills workers AND server.\n" - "For Napari viewers: kills the viewer process.", - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - QMessageBox.StandardButton.No - ) + # Progress-derived nodes are authoritative when present. + if not existing.children and existing.percent <= 0.0: + existing.status = running_status + existing.execution_id = execution_id + if existing.percent <= 0.0: + existing.info = "0.0%" + + for queued in server_info.queued_execution_entries: + plate_id = queued.plate_id + execution_id = queued.execution_id + queue_suffix = f" (q#{queued.queue_position})" + + # Running state is authoritative: do not regress active rows to queued. + if ( + execution_id in running_execution_ids + or plate_id in running_plate_ids + ): + continue + + existing = by_plate_id.get(plate_id) + + if existing is None: + plate_name = self._get_plate_name(plate_id, execution_id) + node = ProgressNode( + node_id=plate_id, + node_type="plate", + label=f"📋 {plate_name}", + status="⏳ Queued", + info=f"0.0%{queue_suffix}", + execution_id=execution_id, + percent=0.0, + children=[], + ) + nodes.append(node) + by_plate_id[plate_id] = node + logger.debug( + f"_merge: created NEW queued node for {plate_id[:30]}..." + ) + continue + + # Progress events are authoritative for the SAME execution. + # For a NEW queued execution (different execution_id), queued overrides. + is_same_execution = existing.execution_id == execution_id + has_real_progress = existing.children or existing.percent > 0 + + if is_same_execution and has_real_progress: + # Same execution with progress - ping lag, keep progress status + logger.debug( + f"_merge: KEEP progress for {plate_id[:30]}... status={existing.status}" + ) + continue + + # Only update to queued if the existing status is not already executing/compiling. + # Progress-derived active status should never be overridden by ping. + if existing.status in ("⚙️ Executing", "⏳ Compiling"): + logger.debug( + f"_merge: SKIP queued for {plate_id[:30]}... already {existing.status}" + ) + continue + + # New queued execution or no progress yet - update to queued + logger.debug( + f"_merge: SET queued for {plate_id[:30]}... (same_exec={is_same_execution})" + ) + existing.status = "⏳ Queued" + existing.execution_id = execution_id + existing.percent = 0.0 + existing.info = f"0.0%{queue_suffix}" + if not is_same_execution: + existing.children = [] + + return nodes + + def _create_tree_item( + self, display: str, status: str, info: str, data: dict + ) -> QTreeWidgetItem: + item = QTreeWidgetItem([display, status, info]) + item.setData(0, Qt.ItemDataRole.UserRole, data) + return item - if reply != QMessageBox.StandardButton.Yes: + @pyqtSlot() + def _refresh_launching_viewers_only(self) -> None: + if self._is_cleaning_up: return - - # Kill in background thread to avoid blocking UI - import threading - - def kill_servers(): - from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG - from zmqruntime.client import ZMQClient - from zmqruntime.queue_tracker import GlobalQueueTrackerRegistry - registry = GlobalQueueTrackerRegistry() - - for port in ports_to_kill: - try: - logger.info(f"🔥 FORCE KILL: Force killing server on port {port} (kills workers AND server)") - # Use kill_server_on_port with graceful=False - # This handles both IPC and TCP modes correctly - success = ZMQClient.kill_server_on_port(port, graceful=False, config=OPENHCS_ZMQ_CONFIG) - - if success: - logger.info(f"✅ Successfully force killed server on port {port}") - else: - logger.warning(f"⚠️ Force kill returned False for port {port}, but continuing cleanup") - - # Clear queue tracker for this viewer (always, regardless of kill result) - registry.remove_tracker(port) - self.server_killed.emit(port) - - except Exception as e: - logger.error(f"❌ Error force killing server on port {port}: {e}") - # Still emit signal to trigger refresh and cleanup, even on error - registry.remove_tracker(port) - self.server_killed.emit(port) - - # Always emit success - we've done our best to kill the processes - # The refresh will remove any dead entries from the list - self._kill_complete.emit(True, "Force kill operation completed (list will refresh)") - - thread = threading.Thread(target=kill_servers, daemon=True) - thread.start() - - def _on_item_double_clicked(self, item: QTreeWidgetItem): - """Handle double-click on tree item - open log file.""" - data = item.data(0, Qt.ItemDataRole.UserRole) - - # For worker items, get the server from parent - if data and data.get('type') == 'worker': - data = data.get('server', {}) - - log_file = data.get('log_file_path') if data else None - - if log_file and Path(log_file).exists(): - # Emit signal for parent to handle (e.g., open in log viewer) - self.log_file_opened.emit(log_file) - logger.info(f"Opened log file: {log_file}") - else: - QMessageBox.information( - self, - "No Log File", - f"No log file available for this item.\n\nPort: {data.get('port', 'unknown') if data else 'unknown'}" - ) + self._update_server_list(self.servers) diff --git a/openhcs/pyqt_gui/widgets/step_parameter_editor.py b/openhcs/pyqt_gui/widgets/step_parameter_editor.py index 9c711a9d9..d9b374cd6 100644 --- a/openhcs/pyqt_gui/widgets/step_parameter_editor.py +++ b/openhcs/pyqt_gui/widgets/step_parameter_editor.py @@ -12,8 +12,15 @@ from pathlib import Path from PyQt6.QtWidgets import ( - QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, - QScrollArea, QSplitter, QTreeWidget, QTreeWidgetItem + QWidget, + QVBoxLayout, + QHBoxLayout, + QPushButton, + QLabel, + QScrollArea, + QSplitter, + QTreeWidget, + QTreeWidgetItem, ) from PyQt6.QtCore import Qt, pyqtSignal, QTimer @@ -30,7 +37,9 @@ from pyqt_reactive.widgets.editors.simple_code_editor import SimpleCodeEditorService from pyqt_reactive.theming import ColorScheme from pyqt_reactive.theming import StyleSheetGenerator +from pyqt_reactive.forms.layout_constants import CURRENT_LAYOUT from openhcs.pyqt_gui.config import PyQtGUIConfig, get_default_pyqt_gui_config + # REMOVED: LazyDataclassFactory import - no longer needed since step editor # uses existing lazy dataclass instances from the step from pyqt_reactive.forms import ParameterTypeUtils @@ -51,38 +60,76 @@ class StepParameterEditorWidget(ScrollableFormMixin, QWidget): Inherits from ScrollableFormMixin to provide scroll-to-section functionality. """ - + # Signals step_parameter_changed = pyqtSignal() - - def __init__(self, step: FunctionStep, service_adapter=None, color_scheme: Optional[ColorScheme] = None, - gui_config: Optional[PyQtGUIConfig] = None, parent=None, pipeline_config=None, scope_id: Optional[str] = None, - step_index: Optional[int] = None): + + def showEvent(self, event): + """Override showEvent to apply initial enabled styling when widget becomes visible.""" + super().showEvent(event) + + def __init__( + self, + step: FunctionStep, + service_adapter=None, + color_scheme: Optional[ColorScheme] = None, + gui_config: Optional[PyQtGUIConfig] = None, + parent=None, + pipeline_config=None, + scope_id: Optional[str] = None, + step_index: Optional[int] = None, + scope_accent_color=None, + render_header: bool = True, + button_style: Optional[str] = None, + ): super().__init__(parent) # Initialize color scheme and GUI config self.color_scheme = color_scheme or ColorScheme() self.gui_config = gui_config or get_default_pyqt_gui_config() self.style_generator = StyleSheetGenerator(self.color_scheme) + self._render_header = render_header + self._button_style = button_style # Store centralized button style + + self.header_label: Optional[QLabel] = None self.step = step self.service_adapter = service_adapter - self.pipeline_config = pipeline_config # Store pipeline config for context hierarchy + self.pipeline_config = ( + pipeline_config # Store pipeline config for context hierarchy + ) self.scope_id = scope_id # Store scope_id for cross-window update scoping self.step_index = step_index # Step position index for tree registry + self.header_label: Optional[QLabel] = None + + # Create action buttons container (always, for external access) + self._action_buttons_container = QWidget() + self._action_buttons_container.setObjectName("step_action_buttons_container") + self._action_buttons_layout = QHBoxLayout(self._action_buttons_container) + self._action_buttons_layout.setContentsMargins(0, 0, 0, 0) + self._action_buttons_layout.setSpacing(2) + self._action_buttons_layout.setAlignment(Qt.AlignmentFlag.AlignRight) + + code_btn = QPushButton("Code") + code_btn.setMaximumWidth(60) + code_btn.setFixedHeight(CURRENT_LAYOUT.button_height) + code_btn.setStyleSheet(self._get_button_style()) + code_btn.clicked.connect(self.view_step_code) + self._action_buttons_layout.addWidget(code_btn) + # Live placeholder updates not yet ready - disable for now self._step_editor_coordinator = None # TODO: Re-enable when live updates feature is fully implemented - # if hasattr(self.gui_config, 'enable_live_step_parameter_updates') and self.gui_config.enable_live_step_parameter_updates: + # if self.gui_config and self.gui_config.enable_live_step_parameter_updates: # from openhcs.config_framework.lazy_factory import ContextEventCoordinator # self._step_editor_coordinator = ContextEventCoordinator() # logger.debug("🔍 STEP EDITOR: Created step-editor-specific coordinator for live step parameter updates") - + # Analyze AbstractStep signature to get all inherited parameters (mirrors Textual TUI) # Auto-detection correctly identifies constructors and includes all parameters param_info = SignatureAnalyzer.analyze(AbstractStep.__init__) - + # Get current parameter values from step instance parameters = {} parameter_types = {} @@ -95,12 +142,14 @@ def __init__(self, step: FunctionStep, service_adapter=None, color_scheme: Optio # CRITICAL FIX: For lazy dataclass parameters, leave current_value as None # This allows the UI to show placeholders and use lazy resolution properly - if current_value is None and self._is_optional_lazy_dataclass_in_pipeline(info.param_type, name): + if current_value is None and self._is_optional_lazy_dataclass_in_pipeline( + info.param_type, name + ): # Don't create concrete instances - leave as None for placeholder resolution # The UI will handle lazy resolution and show appropriate placeholders param_defaults[name] = None # Mark this as a step-level config for special handling - if not hasattr(self, '_step_level_configs'): + if getattr(self, "_step_level_configs", None) is None: self._step_level_configs = {} self._step_level_configs[name] = True else: @@ -110,9 +159,11 @@ def __init__(self, step: FunctionStep, service_adapter=None, color_scheme: Optio parameter_types[name] = info.param_type # Track dataclass-backed parameters for the hierarchy tree - self._tree_dataclass_params = self._collect_dataclass_parameters(parameter_types) + self._tree_dataclass_params = self._collect_dataclass_parameters( + parameter_types + ) self.tree_helper = ConfigHierarchyTreeHelper() - + # SIMPLIFIED: Create parameter form manager using dual-axis resolution # CRITICAL FIX: Use pipeline_config as context_obj (parent for inheritance) @@ -120,9 +171,15 @@ def __init__(self, step: FunctionStep, service_adapter=None, color_scheme: Optio # Context hierarchy: GlobalPipelineConfig (thread-local) -> PipelineConfig (context_obj) -> Step (overlay) # Look up ObjectState from registry using scope_id # ObjectState MUST be registered by PipelineEditorWidget when step was added - logger.info(f"🔍 STEP_EDITOR: Looking up ObjectState for scope_id={self.scope_id}") - logger.info(f"🔍 STEP_EDITOR: Registry has scopes: {[s.scope_id for s in ObjectStateRegistry.get_all()]}") - self.state = ObjectStateRegistry.get_by_scope(self.scope_id) if self.scope_id else None + logger.info( + f"🔍 STEP_EDITOR: Looking up ObjectState for scope_id={self.scope_id}" + ) + logger.info( + f"🔍 STEP_EDITOR: Registry has scopes: {[s.scope_id for s in ObjectStateRegistry.get_all()]}" + ) + self.state = ( + ObjectStateRegistry.get_by_scope(self.scope_id) if self.scope_id else None + ) if self.state is None: # FAIL LOUD: The step MUST be registered by PipelineEditor before opening the editor @@ -132,17 +189,21 @@ def __init__(self, step: FunctionStep, service_adapter=None, color_scheme: Optio f"Registry has: {[s.scope_id for s in ObjectStateRegistry.get_all()]}" ) - logger.info(f"🔍 STEP_EDITOR: Using REGISTERED ObjectState, params={list(self.state.parameters.keys())}") + logger.info( + f"🔍 STEP_EDITOR: Using REGISTERED ObjectState, params={list(self.state.parameters.keys())}" + ) config = FormManagerConfig( - parent=self, # Pass self as parent widget - color_scheme=self.color_scheme, # Pass color scheme for consistent theming - use_scroll_area=False # Step editor manages its own scroll area + parent=self, # Pass self as parent widget + color_scheme=self.color_scheme, # Pass color scheme for consistent theming + use_scroll_area=False, # Step editor manages its own scroll area + scope_accent_color=scope_accent_color, # Pass scope accent color from parent window + scope_step_index=self.step_index, # Align scope styling with pipeline order ) self.form_manager = ParameterFormManager( - state=self.state, # ObjectState (MODEL) from registry - config=config # Pass configuration object + state=self.state, # ObjectState (MODEL) from registry + config=config, # Pass configuration object ) self.hierarchy_tree = None self.content_splitter = None @@ -151,9 +212,21 @@ def __init__(self, step: FunctionStep, service_adapter=None, color_scheme: Optio self.setup_connections() # Ensure placeholders pick up live context (e.g., PipelineConfig edits) after registration. - QTimer.singleShot(0, lambda: ParameterOpsService().refresh_with_live_context(self.form_manager)) + QTimer.singleShot( + 0, + lambda: ParameterOpsService().refresh_with_live_context(self.form_manager), + ) + + logger.debug( + f"Step parameter editor initialized for step: {getattr(step, 'name', 'Unknown')}" + ) + + def apply_scope_color_scheme(self, scheme) -> None: + from pyqt_reactive.widgets.shared.scope_style_applier import ( + apply_scope_color_scheme_to_widget_tree, + ) - logger.debug(f"Step parameter editor initialized for step: {getattr(step, 'name', 'Unknown')}") + apply_scope_color_scheme_to_widget_tree(self.form_manager, scheme) def _is_optional_lazy_dataclass_in_pipeline(self, param_type, param_name): """ @@ -183,8 +256,12 @@ def _is_optional_lazy_dataclass_in_pipeline(self, param_type, param_name): try: # Try to create an instance to see if it's a lazy dataclass test_instance = inner_type() - # Check for lazy dataclass methods - return hasattr(test_instance, '_resolve_field_value') or hasattr(test_instance, '_lazy_resolution_config') + # Check for lazy dataclass methods - direct access will raise AttributeError if missing + test_instance._resolve_field_value + test_instance._lazy_resolution_config + return True + except AttributeError: + return False except: return False @@ -209,7 +286,7 @@ def _collect_dataclass_parameters(self, parameter_types): """Return dataclass-based parameters for building the hierarchy tree.""" dataclass_params = {} for field_name, param_type in parameter_types.items(): - if field_name == 'func': + if field_name == "func": continue obj_type = self._extract_dataclass_from_param_type(param_type) @@ -242,7 +319,7 @@ def _extract_dataclass_from_param_type(self, param_type): def _create_configuration_tree(self) -> Optional[QTreeWidget]: """Create and populate the configuration hierarchy tree.""" - if not getattr(self, '_tree_dataclass_params', None): + if not getattr(self, "_tree_dataclass_params", None): return None # Pass form_manager as flash_manager - tree reads from SAME _flash_colors dict as groupboxes @@ -250,7 +327,8 @@ def _create_configuration_tree(self) -> Optional[QTreeWidget]: # Pass state for automatic dirty tracking subscription (handled by helper) tree = self.tree_helper.create_tree_widget( flash_manager=self.form_manager, - state=self.state + state=self.state, + strip_config_suffix=True, ) self.tree_helper.populate_from_mapping(tree, self._tree_dataclass_params) @@ -266,16 +344,16 @@ def _create_configuration_tree(self) -> Optional[QTreeWidget]: def _on_tree_item_double_clicked(self, item: QTreeWidgetItem, column: int): """Scroll to the associated form section when a tree item is activated.""" data = item.data(0, Qt.ItemDataRole.UserRole) - if not data or data.get('ui_hidden'): + if not data or data.get("ui_hidden"): return - item_type = data.get('type') - if item_type == 'dataclass': - field_name = data.get('field_name') - if field_name: - self._scroll_to_section(field_name) - elif item_type == 'inheritance_link': - target_class = data.get('target_class') + item_type = data.get("type") + if item_type == "dataclass": + field_path = data.get("field_path") or data.get("field_name") + if field_path: + self._scroll_to_section(field_path) + elif item_type == "inheritance_link": + target_class = data.get("target_class") if target_class: field_name = self._find_field_for_class(target_class) if field_name: @@ -291,10 +369,6 @@ def _find_field_for_class(self, target_class) -> Optional[str]: # _scroll_to_section is provided by ScrollableFormMixin - - - - def setup_ui(self): """Setup the user interface (matches FunctionListEditorWidget structure).""" # Main layout directly on self (like FunctionListEditorWidget) @@ -302,42 +376,37 @@ def setup_ui(self): layout.setContentsMargins(0, 0, 0, 0) layout.setSpacing(8) - # Header with controls (like FunctionListEditorWidget) - header_layout = QHBoxLayout() - - # Header label (stored for scope accent styling) - self.header_label = QLabel("Step Parameters") - self.header_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; font-weight: bold; font-size: 14px;") - header_layout.addWidget(self.header_label) - - header_layout.addStretch() + # Header with controls (only if render_header=True) + if self._render_header: + header_layout = QHBoxLayout() - # Action buttons in header (preserving functionality) - code_btn = QPushButton("Code") - code_btn.setMaximumWidth(60) - code_btn.setStyleSheet(self._get_button_style()) - code_btn.clicked.connect(self.view_step_code) - header_layout.addWidget(code_btn) + # Header label (stored for scope accent styling) + self.header_label = QLabel("Step Parameters") + self.header_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; font-weight: bold; font-size: 14px;" + ) + header_layout.addWidget(self.header_label) - load_btn = QPushButton("Load .step") - load_btn.setMaximumWidth(100) - load_btn.setStyleSheet(self._get_button_style()) - load_btn.clicked.connect(self.load_step_settings) - header_layout.addWidget(load_btn) + header_layout.addStretch() - save_btn = QPushButton("Save .step As") - save_btn.setMaximumWidth(120) - save_btn.setStyleSheet(self._get_button_style()) - save_btn.clicked.connect(self.save_step_settings) - header_layout.addWidget(save_btn) + # Add action buttons to header + header_layout.addWidget(self._action_buttons_container) - layout.addLayout(header_layout) + layout.addLayout(header_layout) + else: + # Header not rendered - buttons are still available for external use + # No header layout added, so buttons remain in _action_buttons_container + pass # Scrollable parameter form (matches config window pattern) self.scroll_area = QScrollArea() self.scroll_area.setWidgetResizable(True) - self.scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) - self.scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff) + self.scroll_area.setVerticalScrollBarPolicy( + Qt.ScrollBarPolicy.ScrollBarAsNeeded + ) + self.scroll_area.setHorizontalScrollBarPolicy( + Qt.ScrollBarPolicy.ScrollBarAlwaysOff + ) # No explicit styling - let it inherit from parent # Add form manager directly to scroll area (like config window) @@ -355,7 +424,9 @@ def setup_ui(self): self.content_splitter = splitter # Install collapsible splitter helper for double-click toggle - self.splitter_helper = CollapsibleSplitterHelper(splitter, left_panel_index=0) + self.splitter_helper = CollapsibleSplitterHelper( + splitter, left_panel_index=0 + ) self.splitter_helper.set_initial_size(280) else: layout.addWidget(self.scroll_area) @@ -364,14 +435,35 @@ def setup_ui(self): # Apply tree widget styling (matches config window) self.setStyleSheet(self.style_generator.generate_tree_widget_style()) - + + def get_action_buttons(self) -> Optional[QWidget]: + """Get the action buttons container for external placement. + + This method allows parent windows (e.g., DualEditorWindow) to + extract and reposition action buttons without modifying this widget's + internal structure. + + Returns: + QWidget: Container widget with action buttons (Code, Load, Save). + Returns None if header is rendered (buttons are in use). + """ + if self._render_header: + # Header is rendered, buttons are in use internally + return None + return self._action_buttons_container + def _get_button_style(self) -> str: """Get consistent button styling.""" + if self._button_style: + return self.style_generator.generate_config_button_styles().get( + self._button_style, "" + ) + return """ QPushButton { background-color: {self.color_scheme.to_hex(self.color_scheme.input_bg)}; color: white; - border: 1px solid {self.color_scheme.to_hex(self.color_scheme.border_light)}; + border: none; border-radius: 3px; padding: 6px 12px; font-size: 11px; @@ -383,12 +475,12 @@ def _get_button_style(self) -> str: background-color: {self.color_scheme.to_hex(self.color_scheme.button_pressed_bg)}; } """ - + def setup_connections(self): """Setup signal/slot connections.""" # Connect form manager parameter changes self.form_manager.parameter_changed.connect(self._handle_parameter_change) - + def _handle_parameter_change(self, param_name: str, value: Any): """Handle parameter change from form manager (mirrors Textual TUI). @@ -400,7 +492,7 @@ def _handle_parameter_change(self, param_name: str, value: Any): # Extract leaf field name from full path # "FunctionStep.processing_config.group_by" -> "group_by" # "FunctionStep.name" -> "name" - path_parts = param_name.split('.') + path_parts = param_name.split(".") if len(path_parts) > 1: # Remove type name prefix path_parts = path_parts[1:] @@ -415,9 +507,10 @@ def _handle_parameter_change(self, param_name: str, value: Any): final_value = self.state.get_current_values().get(leaf_field, value) # CRITICAL FIX: For function parameters, use fresh imports to avoid unpicklable registry wrappers - if leaf_field == 'func' and callable(final_value) and hasattr(final_value, '__module__'): + if leaf_field == "func" and callable(final_value): try: import importlib + module = importlib.import_module(final_value.__module__) final_value = getattr(module, final_value.__name__) except Exception: @@ -428,7 +521,9 @@ def _handle_parameter_change(self, param_name: str, value: Any): logger.debug(f"Updated step parameter {leaf_field}={final_value}") else: # Nested field - already updated by _mark_parents_modified - logger.debug(f"Nested field {'.'.join(path_parts)} already updated by dispatcher") + logger.debug( + f"Nested field {'.'.join(path_parts)} already updated by dispatcher" + ) self.step_parameter_changed.emit() @@ -440,47 +535,49 @@ def load_step_settings(self): if not self.service_adapter: logger.warning("No service adapter available for file dialog") return - + file_path = self.service_adapter.show_cached_file_dialog( cache_key=PathCacheKey.STEP_SETTINGS, title="Load Step Settings (.step)", file_filter="Step Files (*.step);;All Files (*)", - mode="open" + mode="open", ) - + if file_path: self._load_step_settings_from_file(file_path) - + def save_step_settings(self): """Save step settings to .step file (mirrors Textual TUI).""" if not self.service_adapter: logger.warning("No service adapter available for file dialog") return - + file_path = self.service_adapter.show_cached_file_dialog( cache_key=PathCacheKey.STEP_SETTINGS, title="Save Step Settings (.step)", file_filter="Step Files (*.step);;All Files (*)", - mode="save" + mode="save", ) - + if file_path: self._save_step_settings_to_file(file_path) - + def _load_step_settings_from_file(self, file_path: Path): """Load step settings from file.""" try: import dill as pickle - with open(file_path, 'rb') as f: + + with open(file_path, "rb") as f: step_data = pickle.load(f) # Update form manager with loaded values for param_name, value in step_data.items(): - if hasattr(self.form_manager, 'update_parameter'): + try: self.form_manager.update_parameter(param_name, value) - # Also update the step object - if hasattr(self.step, param_name): - setattr(self.step, param_name, value) + # Also update to step object + setattr(self.step, param_name, value) + except (AttributeError, AttributeError): + pass # Refresh the form to show loaded values self.form_manager._refresh_all_placeholders() @@ -489,38 +586,44 @@ def _load_step_settings_from_file(self, file_path: Path): except Exception as e: logger.error(f"Failed to load step settings from {file_path}: {e}") if self.service_adapter: - self.service_adapter.show_error_dialog(f"Failed to load step settings: {e}") + self.service_adapter.show_error_dialog( + f"Failed to load step settings: {e}" + ) def _save_step_settings_to_file(self, file_path: Path): """Save step settings to file.""" try: import dill as pickle + # Get current values from state step_data = self.state.get_current_values() - with open(file_path, 'wb') as f: + with open(file_path, "wb") as f: pickle.dump(step_data, f) logger.debug(f"Saved {len(step_data)} parameters to {file_path.name}") except Exception as e: logger.error(f"Failed to save step settings to {file_path}: {e}") if self.service_adapter: - self.service_adapter.show_error_dialog(f"Failed to save step settings: {e}") + self.service_adapter.show_error_dialog( + f"Failed to save step settings: {e}" + ) - def get_current_step(self) -> FunctionStep: """Get the current step with all parameter values.""" return self.step - + def update_step(self, step: FunctionStep): """Update the step and refresh the form.""" self.step = step - + # Update form manager with new values for param_name in self.form_manager.parameters.keys(): current_value = getattr(self.step, param_name, None) self.form_manager.update_parameter(param_name, current_value) - logger.debug(f"Updated step parameter editor for step: {getattr(step, 'name', 'Unknown')}") + logger.debug( + f"Updated step parameter editor for step: {getattr(step, 'name', 'Unknown')}" + ) def view_step_code(self): """View the complete FunctionStep as Python code.""" @@ -535,39 +638,40 @@ def view_step_code(self): current_step = self.state.to_object() # CRITICAL: Get func from parent dual editor's function list editor if available - # The func is managed by the Function Pattern tab in the dual editor + # The func is managed by to Function Pattern tab in the dual editor parent_window = self.window() - if hasattr(parent_window, 'func_editor') and parent_window.func_editor: - # Get live func pattern from function list editor - func = parent_window.func_editor.current_pattern - current_step.func = func - logger.debug(f"Using live func from function list editor: {func}") - else: - logger.debug(f"Using func from step instance: {current_step.func}") + func = parent_window.func_editor.current_pattern + current_step.func = func + logger.debug(f"Using live func from function list editor: {func}") # Generate code using existing pattern python_code = generate_python_source( Assignment("step", current_step), header="# Function Step", - clean_mode=False, + clean_mode=True, ) # Launch editor editor_service = SimpleCodeEditorService(self) - use_external = os.environ.get('OPENHCS_USE_EXTERNAL_EDITOR', '').lower() in ('1', 'true', 'yes') + use_external = os.environ.get( + "OPENHCS_USE_EXTERNAL_EDITOR", "" + ).lower() in ("1", "true", "yes") editor_service.edit_code( initial_content=python_code, title=f"Edit Step: {current_step.name}", callback=self._handle_edited_step_code, use_external=use_external, - code_type='step' + code_type="step", + code_data={"clean_mode": True}, ) except Exception as e: logger.error(f"Failed to open step code editor: {e}") if self.service_adapter: - self.service_adapter.show_error_dialog(f"Failed to open code editor: {str(e)}") + self.service_adapter.show_error_dialog( + f"Failed to open code editor: {str(e)}" + ) def _handle_edited_step_code(self, edited_code: str) -> None: """Handle the edited step code from code editor.""" @@ -579,7 +683,7 @@ def _handle_edited_step_code(self, edited_code: str) -> None: with CodeEditorFormUpdater.patch_lazy_constructors(): exec(edited_code, namespace) - new_step = namespace.get('step') + new_step = namespace.get("step") if not new_step: raise ValueError("No 'step' variable found in edited code") @@ -599,11 +703,10 @@ def _handle_edited_step_code(self, edited_code: str) -> None: # CRITICAL: Update function list editor if we're inside a dual editor window parent_window = self.window() - if hasattr(parent_window, 'func_editor') and parent_window.func_editor: - func_editor = parent_window.func_editor - func_editor._initialize_pattern_data(new_step.func) - func_editor._populate_function_list() - logger.debug(f"Updated function list editor with new func: {new_step.func}") + func_editor = parent_window.func_editor + func_editor._initialize_pattern_data(new_step.func) + func_editor._populate_function_list() + logger.debug(f"Updated function list editor with new func: {new_step.func}") # Notify parent window that step parameters changed self.step_parameter_changed.emit() diff --git a/openhcs/pyqt_gui/windows/__init__.py b/openhcs/pyqt_gui/windows/__init__.py index ed8d6e811..6832cf05e 100644 --- a/openhcs/pyqt_gui/windows/__init__.py +++ b/openhcs/pyqt_gui/windows/__init__.py @@ -1,22 +1,34 @@ """ OpenHCS PyQt6 Windows -Window components for the OpenHCS PyQt6 GUI application. +Window components for OpenHCS PyQt6 GUI application. All windows migrated from Textual TUI with full feature parity. """ -from openhcs.pyqt_gui.windows.base_form_dialog import BaseFormDialog +from pyqt_reactive.widgets.shared import BaseFormDialog from openhcs.pyqt_gui.windows.config_window import ConfigWindow from openhcs.pyqt_gui.windows.help_window import HelpWindow -from openhcs.pyqt_gui.windows.dual_editor_window import DualEditorWindow from openhcs.pyqt_gui.windows.file_browser_window import FileBrowserWindow -from openhcs.pyqt_gui.windows.synthetic_plate_generator_window import SyntheticPlateGeneratorWindow +from openhcs.pyqt_gui.windows.synthetic_plate_generator_window import ( + SyntheticPlateGeneratorWindow, +) +from openhcs.pyqt_gui.windows.managed_windows import ( + PlateManagerWindow, + PipelineEditorWindow, + ImageBrowserWindow, + LogViewerWindowWrapper, + ZMQServerManagerWindow, +) __all__ = [ "BaseFormDialog", "ConfigWindow", "HelpWindow", - "DualEditorWindow", "FileBrowserWindow", - "SyntheticPlateGeneratorWindow" + "SyntheticPlateGeneratorWindow", + "PlateManagerWindow", + "PipelineEditorWindow", + "ImageBrowserWindow", + "LogViewerWindowWrapper", + "ZMQServerManagerWindow", ] diff --git a/openhcs/pyqt_gui/windows/base_form_dialog.py b/openhcs/pyqt_gui/windows/base_form_dialog.py deleted file mode 100644 index 6c05e9467..000000000 --- a/openhcs/pyqt_gui/windows/base_form_dialog.py +++ /dev/null @@ -1,589 +0,0 @@ -""" -Base Form Dialog for PyQt6 - -Base class for dialogs that use ParameterFormManager to ensure proper cleanup -of cross-window placeholder update connections. - -This base class solves the problem of ghost form managers remaining in -_active_form_managers registry after a dialog closes, which causes infinite -placeholder refresh loops and runaway CPU usage. - -The issue occurs because Qt's QDialog.accept() and QDialog.reject() methods -do NOT trigger closeEvent() - they just hide the dialog. This means any cleanup -code in closeEvent() is never called when the user clicks Save or Cancel. - -This base class overrides accept(), reject(), and closeEvent() to ensure that -form managers are always unregistered from cross-window updates, regardless of -how the dialog is closed. - -The default implementation automatically discovers all ParameterFormManager -instances in the widget tree, so subclasses don't need to manually track them. - - -Usage: - 1. Inherit from BaseFormDialog instead of QDialog - 2. That's it! All ParameterFormManager instances are automatically discovered and cleaned up. - -Example: - class MyConfigDialog(BaseFormDialog): - def __init__(self, ...): - super().__init__(...) - self.form_manager = ParameterFormManager(...) - # No need to override _get_form_managers() - automatic discovery! - """ -import logging -from typing import Optional, Protocol, Callable - -from PyQt6.QtCore import QEvent, Qt -from PyQt6.QtWidgets import QApplication, QDialog, QPushButton, QWidget - -from pyqt_reactive.animation import WindowFlashOverlay -from pyqt_reactive.services.parameter_ops_service import ParameterOpsService -from pyqt_reactive.widgets.function_list_editor import FunctionListEditorWidget -from pyqt_reactive.widgets.function_pane import FunctionPaneWidget -from pyqt_reactive.widgets.shared.clickable_help_components import ( - HelpButton, - HelpIndicator, - GroupBoxWithHelp, - ) -from pyqt_reactive.widgets.shared.config_hierarchy_tree import ConfigHierarchyTreeHelper -from openhcs.config_framework.object_state import ObjectStateRegistry -from pyqt_reactive.services.window_manager import WindowManager - -# Import ScopedBorderMixin directly from module, avoiding widgets/__init__.py (circular import) -import pyqt_reactive.widgets.shared.scoped_border_mixin as _scoped_border_module -ScopedBorderMixin = _scoped_border_module.ScopedBorderMixin - -logger = logging.getLogger(__name__) - - -class HasUnregisterMethod(Protocol): - """Protocol for objects that can be unregistered from cross-window updates.""" - def unregister_from_cross_window_updates(self) -> None: ... - - -class BaseFormDialog(ScopedBorderMixin, QDialog): - """Base dialog with automatic singleton-per-scope behavior. - - Ensures only ONE window per (scope_id, window_class) is open at a time. - If a window for this scope already exists, show() focuses it instead of creating a duplicate. - """ - def __init__(self, parent=None): - super().__init__(parent) - self._unregistered = False # Track if we've already unregistered - self._flash_overlay_cleaned = False # Track overlay cleanup when dialog is reused - self._scope_accent_color = None # Stored for deferred application to async widgets - - # CRITICAL: Register with global event bus for cross-window updates - # This is the OpenHCS "set and forget" pattern - all windows automatically - # receive updates from all other windows without manual connections - event_bus = self._get_event_bus() - if event_bus: - event_bus.register_window(self) - # Connect to pipeline_changed signal if this window cares about pipeline updates - if hasattr(self, '_on_pipeline_changed'): - event_bus.pipeline_changed.connect(self._on_pipeline_changed) - logger.debug(f"{self.__class__.__name__}: Registered with global event bus") - - def show(self) -> None: - """Override show to enforce singleton-per-scope behavior. - - If a window with the same scope_key already exists, focus it instead of showing self. - Otherwise, register self with WindowManager and show normally. - - Also initializes scope-based border styling here (not in __init__) because - subclasses set scope_id AFTER calling super().__init__(). - """ - scope_key = self._get_window_scope_key() - if scope_key is None: - # No scope_id - just show normally (legacy behavior) - super().show() - return - - # Check if window already exists for this scope - if WindowManager.is_open(scope_key): - # Focus existing window instead of showing duplicate - WindowManager.focus_and_navigate(scope_key) - # DON'T call deleteLater() - it causes crashes because async widget creation - # (QTimer.singleShot) continues running and references deleted objects. - # Instead, just return without showing. The window will be garbage collected - # naturally when all async work completes and references are released. - logger.debug(f"[SINGLETON] Focused existing window for {scope_key}") - return - - overlay_was_cleaned = getattr(self, "_flash_overlay_cleaned", False) - - # Initialize scope-based border styling (ScopedBorderMixin) - # Done here because scope_id is set by subclass AFTER super().__init__() - self._init_scope_border() - - # Register self with WindowManager (it will handle closeEvent cleanup) - # NOTE: WindowManager.register() also eagerly creates a flash overlay - WindowManager.register(scope_key, self) - super().show() - if overlay_was_cleaned: - self._reregister_flash_elements() - self._flash_overlay_cleaned = False - logger.debug(f"[SINGLETON] Registered and showed new window for {scope_key}") - - def _get_window_scope_key(self) -> Optional[str]: - """Build unique key for WindowManager registration. - - Returns: - The scope_id directly (e.g., "", "/path/to/plate", "/plate::step_0") - Returns None if no scope_id is set (legacy behavior - no singleton enforcement) - - Note: We use scope_id directly (not scope_id::ClassName) so that provenance - navigation can find windows by scope_id. The scope_id is already unique - per context (global, plate, step, function). - """ - scope_id = getattr(self, 'scope_id', None) - if scope_id is None: - return None - - return scope_id - - def _setup_save_button(self, button: 'QPushButton', save_callback: Callable): - """ - Connects a save button to support Shift+Click for 'Save without close'. - - The save_callback should accept only close_window as a keyword argument. - If Shift is held, close_window will be False (update only); otherwise True. - """ - def _on_save(): - modifiers = QApplication.keyboardModifiers() - is_shift = modifiers & Qt.KeyboardModifier.ShiftModifier - save_callback(close_window=not is_shift) - button.clicked.connect(_on_save) - - def _mark_saved_and_refresh_all(self): - """Mark all states as saved and refresh placeholders. - - Used by shift-click save to update saved baseline without closing window. - Regular save calls accept() which also marks saved but additionally closes window. - - CRITICAL: This ensures shift-click save works the same as regular save: - - Marks all states as saved (reconstructs object_instance from ORM values via to_object()) - - Updates saved baseline so closing window later won't revert changes - - Refreshes all form managers with new saved values - - Notifies other windows of changes - """ - # Mark all states as saved (updates object_instance from ORM values) - self._apply_state_action('mark_saved') - - # Increment global token to invalidate all caches - ObjectStateRegistry.increment_token() - - # Refresh all form managers with new saved values as baseline - for manager in self._get_form_managers(): - ParameterOpsService().refresh_with_live_context(manager) - # Emit context_changed to notify other windows (bulk refresh) - manager.context_changed.emit(manager.scope_id or "", "") - - def _apply_state_action(self, action: str) -> None: - """Apply a state-level action (mark_saved/restore_saved) to all discovered managers.""" - if action not in ("mark_saved", "restore_saved"): - return - - seen_states = set() - for manager in self._get_form_managers(): - try: - state = manager.state - except AttributeError: - continue - - state_id = id(state) - if state_id in seen_states: - continue - - try: - field_id = getattr(state, 'field_id', '?') - scope_id = getattr(state, 'scope_id', '?') - obj_type = type(state.object_instance).__name__ if hasattr(state, 'object_instance') else '?' - logger.debug(f"🐛 {self.__class__.__name__}: About to call {action} on state field_id={field_id!r}, scope={scope_id!r}, obj_type={obj_type}") - logger.debug(f"🐛 {self.__class__.__name__}: State has _saved_parameters={hasattr(state, '_saved_parameters')}, is None={getattr(state, '_saved_parameters', 'N/A') is None}") - if action == "mark_saved": - state.mark_saved() - else: - state.restore_saved() - except Exception as e: - import traceback - field_id = getattr(state, 'field_id', '?') - logger.warning(f"{self.__class__.__name__}: failed to {action} for state {field_id}: {e}") - logger.warning(f"🐛 Full traceback:\n{traceback.format_exc()}") - - seen_states.add(state_id) - - def _get_event_bus(self): - """Get the global event bus from the service adapter. - - Returns: - GlobalEventBus instance or None if not found - """ - try: - # Navigate up to find main window with service adapter - current = self.parent() - while current: - if hasattr(current, 'service_adapter'): - return current.service_adapter.get_event_bus() - current = current.parent() - logger.debug(f"{self.__class__.__name__}: Could not find service adapter for event bus") - return None - except Exception as e: - logger.error(f"Error getting event bus: {e}") - return None - - def _broadcast_pipeline_changed(self, pipeline_steps: list): - """Broadcast pipeline changed event to all windows via event bus. - - Args: - pipeline_steps: Updated list of FunctionStep objects - """ - event_bus = self._get_event_bus() - if event_bus: - event_bus.emit_pipeline_changed(pipeline_steps) - - def _broadcast_config_changed(self, config): - """Broadcast config changed event to all windows via event bus. - - Args: - config: Updated config object - """ - event_bus = self._get_event_bus() - if event_bus: - event_bus.emit_config_changed(config) - - def _get_form_managers(self): - """ - Return a list of all ParameterFormManager instances that need to be unregistered. - - Default implementation recursively searches the widget tree for all - ParameterFormManager instances. Subclasses can override for custom behavior. - - Returns: - List of ParameterFormManager instances - """ - managers = [] - self._collect_form_managers_recursive(self, managers, visited=set()) - return managers - - def _collect_form_managers_recursive(self, widget, managers, visited): - """ - Recursively collect all ParameterFormManager instances from widget tree. - - This eliminates the need for manual tracking - just inherit from BaseFormDialog - and all nested form managers will be automatically discovered and cleaned up. - - Uses Protocol-based duck typing to check for unregister method, avoiding - hasattr smell for guaranteed attributes while still supporting dynamic discovery. - """ - # Prevent infinite loops from circular references - widget_id = id(widget) - if widget_id in visited: - return - - visited.add(widget_id) - - # Check if this widget IS a ParameterFormManager (duck typing via Protocol) - # This is legitimate hasattr - we're discovering unknown widget types - if callable(getattr(widget, 'unregister_from_cross_window_updates', None)): - managers.append(widget) - return # Don't recurse into the manager itself - - # Check if this widget HAS a form_manager attribute - # This is legitimate - form_manager is an optional composition pattern - form_manager = getattr(widget, 'form_manager', None) - if form_manager is not None and callable(getattr(form_manager, 'unregister_from_cross_window_updates', None)): - managers.append(form_manager) - - # Recursively search child widgets using Qt's children() method - try: - for child in widget.children(): - self._collect_form_managers_recursive(child, managers, visited) - except (RuntimeError, AttributeError): - # Widget already deleted - this is expected during cleanup - pass - - # Also check common container attributes that might hold widgets - # These are known patterns in our UI architecture - for attr_name in ['function_panes', 'step_editor', 'func_editor', 'parameter_editor']: - attr_value = getattr(widget, attr_name, None) - if attr_value is not None: - # Handle lists of widgets - if isinstance(attr_value, list): - for item in attr_value: - self._collect_form_managers_recursive(item, managers, visited) - # Handle single widget - else: - self._collect_form_managers_recursive(attr_value, managers, visited) - - def _apply_scope_accent_styling(self) -> None: - """Apply scope accent color to all UI elements. - - Called by ScopedBorderMixin._init_scope_border() after scope color scheme is set. - Styles: input focus borders, HelpButtons. Subclasses can extend for additional elements. - """ - accent_color = self.get_scope_accent_color() - if not accent_color: - return - - # Store for deferred application to async-created widgets - self._scope_accent_color = accent_color - hex_color = accent_color.name() - - # Apply window-level stylesheet for input widget focus borders - input_focus_style = f""" - QLineEdit:focus {{ - border: 2px solid {hex_color}; - }} - QComboBox:focus {{ - border: 2px solid {hex_color}; - }} - QSpinBox:focus {{ - border: 2px solid {hex_color}; - }} - QDoubleSpinBox:focus {{ - border: 2px solid {hex_color}; - }} - """ - current_window_style = self.styleSheet() or "" - self.setStyleSheet(f"{current_window_style}\n{input_focus_style}") - - # Apply to existing HelpButtons now - self._apply_accent_to_help_buttons() - - # Register callback for async-created widgets using existing infrastructure - for manager in self._get_form_managers(): - if hasattr(manager, '_on_build_complete_callbacks'): - manager._on_build_complete_callbacks.append(self._apply_accent_to_help_buttons) - - def _apply_accent_to_help_buttons(self) -> None: - """Apply scope accent color to all HelpButton, HelpIndicator, GroupBoxWithHelp, and FunctionPaneWidget, and tree instances.""" - accent_color = getattr(self, '_scope_accent_color', None) - if not accent_color: - return - - # Find all HelpButtons, HelpIndicators, GroupBoxWithHelp, and FunctionPaneWidget in entire dialog - if isinstance(self, QWidget): - for help_btn in self.findChildren(HelpButton): - help_btn.set_scope_accent_color(accent_color) - for help_indicator in self.findChildren(HelpIndicator): - help_indicator.set_scope_accent_color(accent_color) - # Apply scope border pattern to nested groupboxes - scope_scheme = getattr(self, '_scope_color_scheme', None) - if scope_scheme: - for groupbox in self.findChildren(GroupBoxWithHelp): - groupbox.set_scope_color_scheme(scope_scheme) - # FunctionPaneWidget needs scope accent for title color (no border) - for func_pane in self.findChildren(FunctionPaneWidget): - func_pane.set_scope_color_scheme(scope_scheme) - # FunctionListEditorWidget needs scheme to apply to newly created panes - for func_editor in self.findChildren(FunctionListEditorWidget): - func_editor.set_scope_color_scheme(scope_scheme) - # Apply scope background to hierarchy tree (step editor) - self._apply_scope_to_hierarchy_trees(scope_scheme) - - def _apply_scope_to_hierarchy_trees(self, scope_scheme) -> None: - """Apply scope-colored background to hierarchy trees in form managers.""" - for manager in self._get_form_managers(): - # Get tree from parent widget (step_editor has hierarchy_tree) - parent = getattr(manager, 'parent', None) - if parent and callable(parent): - parent = parent() - tree = getattr(parent, 'hierarchy_tree', None) - tree_helper = getattr(parent, 'tree_helper', None) - if tree and tree_helper and isinstance(tree_helper, ConfigHierarchyTreeHelper): - tree_helper.apply_scope_background(tree, scope_scheme) - - def _apply_state_action(self, action: str) -> None: - """Apply a state-level action (mark_saved/restore_saved) to all discovered managers.""" - if action not in ("mark_saved", "restore_saved"): - return - - seen_states = set() - for manager in self._get_form_managers(): - try: - state = manager.state - except AttributeError: - continue - - state_id = id(state) - if state_id in seen_states: - continue - - try: - field_id = getattr(state, 'field_id', '?') - scope_id = getattr(state, 'scope_id', '?') - obj_type = type(state.object_instance).__name__ if hasattr(state, 'object_instance') else '?' - logger.debug(f"🐛 {self.__class__.__name__}: About to call {action} on state field_id={field_id!r}, scope={scope_id!r}, obj_type={obj_type}") - logger.debug(f"🐛 {self.__class__.__name__}: State has _saved_parameters={hasattr(state, '_saved_parameters')}, is None={getattr(state, '_saved_parameters', 'N/A') is None}") - if action == "mark_saved": - state.mark_saved() - else: - state.restore_saved() - except Exception as e: - import traceback - field_id = getattr(state, 'field_id', '?') - logger.warning(f"{self.__class__.__name__}: failed to {action} for state {field_id}: {e}") - logger.warning(f"🐛 Full traceback:\n{traceback.format_exc()}") - - seen_states.add(state_id) - - def _get_event_bus(self): - """Get the global event bus from the service adapter. - - Returns: - GlobalEventBus instance or None if not found - """ - try: - # Navigate up to find main window with service adapter - current = self.parent() - while current: - if hasattr(current, 'service_adapter'): - return current.service_adapter.get_event_bus() - current = current.parent() - logger.debug(f"{self.__class__.__name__}: Could not find service adapter for event bus") - return None - except Exception as e: - logger.error(f"Error getting event bus: {e}") - return None - - def _broadcast_pipeline_changed(self, pipeline_steps: list): - """Broadcast pipeline changed event to all windows via event bus. - - Args: - pipeline_steps: Updated list of FunctionStep objects - """ - event_bus = self._get_event_bus() - if event_bus: - event_bus.emit_pipeline_changed(pipeline_steps) - - def _broadcast_config_changed(self, config): - """Broadcast config changed event to all windows via event bus. - - Args: - config: Updated config object - """ - event_bus = self._get_event_bus() - if event_bus: - event_bus.emit_config_changed(config) - - def _unregister_all_form_managers(self): - """Unregister all form managers from cross-window updates.""" - if self._unregistered: - logger.debug(f"🔍 {self.__class__.__name__}: Already unregistered, skipping") - return - - logger.info(f"🔍 {self.__class__.__name__}: Unregistering all form managers") - - # Unregister from event bus - event_bus = self._get_event_bus() - if event_bus: - event_bus.unregister_window(self) - logger.debug(f"{self.__class__.__name__}: Unregistered from global event bus") - - managers = self._get_form_managers() - if not managers: - logger.debug(f"🔍 {self.__class__.__name__}: No form managers found to unregister") - return - - for manager in managers: - try: - logger.info(f"🔍 {self.__class__.__name__}: Calling unregister on {manager.field_id} (id={id(manager)})") - manager.unregister_from_cross_window_updates() - except Exception as e: - logger.error(f"Failed to unregister form manager {manager.field_id}: {e}") - - self._unregistered = True - logger.info(f"🔍 {self.__class__.__name__}: All form managers unregistered") - - def accept(self): - """Override accept to unregister before closing.""" - logger.info(f"🔍 {self.__class__.__name__}: accept() called") - # Persist current form values as the new saved baseline - self._apply_state_action('mark_saved') - self._unregister_all_form_managers() - - # CRITICAL: Cleanup WindowFlashOverlay to prevent memory leak - self._flash_overlay_cleaned = True - WindowFlashOverlay.cleanup_window(self) - logger.info(f"🔍 {self.__class__.__name__}: Cleaned up WindowFlashOverlay") - - super().accept() - - def reject(self): - """Override reject to unregister before closing.""" - logger.info(f"🔍 {self.__class__.__name__}: reject() called") - - # CRITICAL FIX: Only restore saved state if there are unsaved changes - # Check if there are unsaved changes that need to be reverted - has_unsaved_changes = False - for manager in self._get_form_managers(): - try: - state = manager.state - if hasattr(state, 'dirty_fields') and state.dirty_fields: - has_unsaved_changes = True - break - except AttributeError: - continue - - if has_unsaved_changes: - # Revert to last saved baseline before closing - self._apply_state_action('restore_saved') - else: - # No unsaved changes - don't restore, just unregister - logger.debug(f"🔍 {self.__class__.__name__}: No unsaved changes, skipping restore_saved()") - - self._unregister_all_form_managers() - - # CRITICAL: Cleanup WindowFlashOverlay to prevent memory leak - self._flash_overlay_cleaned = True - WindowFlashOverlay.cleanup_window(self) - logger.info(f"🔍 {self.__class__.__name__}: Cleaned up WindowFlashOverlay") - - super().reject() - - def closeEvent(self, a0): - """Override closeEvent to unregister before closing. - - When user closes via X button (not via accept/reject), we need to: - 1. Restore saved state for any unsaved changes - 2. Trigger global refresh so other windows sync - 3. Cleanup WindowFlashOverlay to prevent memory leak - 4. Unregister from WindowManager for singleton tracking - """ - logger.info(f"🔍 {self.__class__.__name__}: closeEvent() called") - - # Restore saved state (reverts unsaved changes) - # This is safe even if no changes - restore_saved() is idempotent - self._apply_state_action('restore_saved') - self._unregister_all_form_managers() - - # CRITICAL: Cleanup WindowFlashOverlay to prevent memory leak - self._flash_overlay_cleaned = True - WindowFlashOverlay.cleanup_window(self) - logger.info(f"🔍 {self.__class__.__name__}: Cleaned up WindowFlashOverlay") - - # Unregister from WindowManager so window can be reopened - scope_key = self._get_window_scope_key() - # CRITICAL: Use "is not None" check, not truthiness! - # scope_key="" (empty string) is a valid scope for GlobalPipelineConfig - if scope_key is not None: - WindowManager.unregister(scope_key) - logger.debug(f"🔍 {self.__class__.__name__}: Unregistered from WindowManager: {scope_key}") - - super().closeEvent(a0) - - # Trigger global refresh AFTER unregistration so other windows - # re-collect live context without this window's cancelled values - ObjectStateRegistry.increment_token() - logger.info(f"🔍 {self.__class__.__name__}: Triggered global refresh after closeEvent") - - def _reregister_flash_elements(self): - """Re-register flashable widgets after overlay cleanup when reusing dialog instance.""" - for manager in self._get_form_managers(): - try: - if hasattr(manager, "reregister_flash_elements"): - manager.reregister_flash_elements() - except Exception as exc: - logger.warning(f"{self.__class__.__name__}: Failed to re-register flash elements for manager {getattr(manager, 'field_id', '?')}: {exc}") diff --git a/openhcs/pyqt_gui/windows/config_window.py b/openhcs/pyqt_gui/windows/config_window.py index 4666ead8c..fc2b17b1f 100644 --- a/openhcs/pyqt_gui/windows/config_window.py +++ b/openhcs/pyqt_gui/windows/config_window.py @@ -12,15 +12,30 @@ from typing import Type, Any, Callable, Optional, Dict from PyQt6.QtWidgets import ( - QDialog, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, - QScrollArea, QWidget, QSplitter, QTreeWidget, QTreeWidgetItem, - QLineEdit, QSpinBox, QDoubleSpinBox, QCheckBox, QComboBox, QMessageBox + QDialog, + QVBoxLayout, + QHBoxLayout, + QPushButton, + QLabel, + QScrollArea, + QWidget, + QSplitter, + QTreeWidget, + QTreeWidgetItem, + QLineEdit, + QSpinBox, + QDoubleSpinBox, + QCheckBox, + QComboBox, + QMessageBox, + QSizePolicy, ) from PyQt6.QtCore import Qt, pyqtSignal, QTimer -from PyQt6.QtGui import QFont +from PyQt6.QtGui import QFont, QShowEvent # Infrastructure classes removed - functionality migrated to ParameterFormManager service layer from pyqt_reactive.forms import ParameterFormManager, FormManagerConfig +from pyqt_reactive.forms.layout_constants import CURRENT_LAYOUT from pyqt_reactive.widgets.shared.config_hierarchy_tree import ConfigHierarchyTreeHelper from pyqt_reactive.widgets.shared.scrollable_form_mixin import ScrollableFormMixin from pyqt_reactive.core.collapsible_splitter_helper import CollapsibleSplitterHelper @@ -29,7 +44,10 @@ from pyqt_reactive.widgets.editors.simple_code_editor import SimpleCodeEditorService from pyqt_reactive.theming import StyleSheetGenerator from pyqt_reactive.theming import ColorScheme -from openhcs.pyqt_gui.windows.base_form_dialog import BaseFormDialog +from pyqt_reactive.widgets.shared import ( + BaseFormDialog, + StagedWrapLayout, +) from openhcs.core.config import GlobalPipelineConfig, PipelineConfig from openhcs.config_framework import is_global_config_type from openhcs.config_framework.global_config import ( @@ -41,6 +59,7 @@ from pycodify import Assignment, generate_python_source from openhcs.ui.shared.code_editor_form_updater import CodeEditorFormUpdater from openhcs.config_framework.object_state import ObjectState, ObjectStateRegistry + # ❌ REMOVED: require_config_context decorator - enhanced decorator events system handles context automatically from openhcs.core.lazy_placeholder import ( LazyDefaultPlaceholderService as FullLazyDefaultPlaceholderService, @@ -50,10 +69,113 @@ ) - logger = logging.getLogger(__name__) +class _StagedButtonWrap(QWidget): + def __init__(self, parent=None, spacing=4): + super().__init__(parent) + self._spacing = spacing + self._groups = [] + self._stay_priority = [] + self._right_align_names = set() + self._last_row1 = [] + self._last_row2 = [] + self._last_width = -1 + + self._resize_timer = QTimer(self) + self._resize_timer.setSingleShot(True) + self._resize_timer.timeout.connect(self._update_layout) + + self._main_layout = QVBoxLayout(self) + self._main_layout.setContentsMargins(0, 0, 0, 0) + self._main_layout.setSpacing(spacing) + + self._row1_widget = QWidget(self) + self._row1_layout = QHBoxLayout(self._row1_widget) + self._row1_layout.setContentsMargins(0, 0, 0, 0) + self._row1_layout.setSpacing(spacing) + self._main_layout.addWidget(self._row1_widget) + + self._row2_widget = QWidget(self) + self._row2_layout = QHBoxLayout(self._row2_widget) + self._row2_layout.setContentsMargins(0, 0, 0, 0) + self._row2_layout.setSpacing(spacing) + self._main_layout.addWidget(self._row2_widget) + self._row2_widget.hide() + + def set_groups(self, groups, stay_priority, right_align_names=None): + self._groups = groups + self._stay_priority = stay_priority + self._right_align_names = set(right_align_names or []) + self._update_layout() + + def resizeEvent(self, a0): + super().resizeEvent(a0) + self._resize_timer.start(50) + + def _clear_row(self, layout): + while layout.count(): + item = layout.takeAt(0) + if item and item.widget(): + item.widget().setParent(None) + + def _row_width(self, names, widths): + if not names: + return 0 + total = 0 + for name in names: + total += widths.get(name, 0) + total += self._spacing * (len(names) - 1) + return total + + def _update_layout(self): + if not self._groups: + return + + available = self.width() + visual_order = [name for name, _ in self._groups] + widths = {name: widget.sizeHint().width() for name, widget in self._groups} + + keep_names = [] + for name in self._stay_priority: + candidate = keep_names + [name] + if available <= 0 or self._row_width(candidate, widths) <= available: + keep_names.append(name) + + row1_names = [name for name in visual_order if name in keep_names] + row2_names = [name for name in visual_order if name not in keep_names] + + if ( + available == self._last_width + and row1_names == self._last_row1 + and row2_names == self._last_row2 + ): + return + + self._last_row1 = list(row1_names) + self._last_row2 = list(row2_names) + self._last_width = available + + group_map = {name: widget for name, widget in self._groups} + + self._clear_row(self._row1_layout) + for name in row1_names: + self._row1_layout.addWidget(group_map[name]) + + self._clear_row(self._row2_layout) + row2_left = [name for name in row2_names if name not in self._right_align_names] + row2_right = [name for name in row2_names if name in self._right_align_names] + for name in row2_left: + self._row2_layout.addWidget(group_map[name]) + if row2_right: + self._row2_layout.addStretch(1) + for name in row2_right: + self._row2_layout.addWidget(group_map[name]) + + self._row2_widget.setVisible(bool(row2_names)) + + # Infrastructure classes removed - functionality migrated to ParameterFormManager service layer @@ -75,11 +197,17 @@ class ConfigWindow(ScrollableFormMixin, BaseFormDialog): # Signals config_saved = pyqtSignal(object) # saved config config_cancelled = pyqtSignal() - - def __init__(self, config_class: Type, current_config: Any, - on_save_callback: Optional[Callable] = None, - color_scheme: Optional[ColorScheme] = None, parent=None, - scope_id: Optional[str] = None): + changes_detected = pyqtSignal(bool) # has_changes + + def __init__( + self, + config_class: Type, + current_config: Any, + on_save_callback: Optional[Callable] = None, + color_scheme: Optional[ColorScheme] = None, + parent=None, + scope_id: Optional[str] = None, + ): """ Initialize the configuration window. @@ -106,18 +234,28 @@ def __init__(self, config_class: Type, current_config: Any, self._suppress_global_context_sync = False self._needs_global_context_resync = False + # Change tracking + self.has_changes = False + # Initialize color scheme and style generator self.color_scheme = color_scheme or ColorScheme() self.style_generator = StyleSheetGenerator(self.color_scheme) self.tree_helper = ConfigHierarchyTreeHelper() + # NOTE: _init_scope_border() will be called AFTER setup_ui() creates the widgets + # This ensures widgets exist when _apply_scope_accent_styling() tries to style them + # SIMPLIFIED: Use dual-axis resolution # Determine placeholder prefix based on actual instance type (not class type) - is_lazy_dataclass = FullLazyDefaultPlaceholderService.has_lazy_resolution(type(current_config)) + is_lazy_dataclass = FullLazyDefaultPlaceholderService.has_lazy_resolution( + type(current_config) + ) placeholder_prefix = "Pipeline default" if is_lazy_dataclass else "Default" # SIMPLIFIED: Use ParameterFormManager with dual-axis resolution - root_field_id = type(current_config).__name__ # e.g., "GlobalPipelineConfig" or "PipelineConfig" + root_field_id = type( + current_config + ).__name__ # e.g., "GlobalPipelineConfig" or "PipelineConfig" global_config_type = GlobalPipelineConfig # Always use GlobalPipelineConfig for dual-axis resolution # CRITICAL FIX: Pipeline Config Editor should NOT use itself as parent context @@ -133,18 +271,36 @@ def __init__(self, config_class: Type, current_config: Any, scope_id=self.scope_id, ) + # When editing per-orchestrator PipelineConfig we typically reuse the orchestrator's + # ObjectState (delegated to pipeline_config) under the plate scope_id. + # On Cancel/close we want to restore the PipelineConfig fields, but NOT restore + # descendant step ObjectStates (which can clear the visible pipeline). + if ( + self.config_class is PipelineConfig + and self.scope_id not in (None, "") + and getattr(self.state, "_delegate_attr", None) is not None + ): + self._restore_descendants_on_close = False + # CRITICAL: Config window manages its own scroll area, so tell form_manager NOT to create one config = FormManagerConfig( parent=None, scope_id=self.scope_id, color_scheme=self.color_scheme, - use_scroll_area=False, # Config window handles scrolling + scope_accent_color=getattr( + self, "_scope_accent_color", None + ), # Pass scope accent color ) + # Provide canonical dotted `field_id` for this root form + # Root forms use an empty `field_id` (top-level) so no traversal is attempted + config.field_id = "" self.form_manager = ParameterFormManager(state=self.state, config=config) if is_global_config_type(self.config_class): self._original_global_config_snapshot = copy.deepcopy(current_config) - self.form_manager.parameter_changed.connect(self._on_global_config_field_changed) + self.form_manager.parameter_changed.connect( + self._on_global_config_field_changed + ) # No config_editor needed - everything goes through form_manager self.config_editor = None @@ -154,9 +310,20 @@ def __init__(self, config_class: Type, current_config: Any, self._dirty_title_callback = self._update_window_title_dirty_marker self.state.on_state_changed(self._dirty_title_callback) + # Change detection + self.changes_detected.connect(self.on_changes_detected) + # Setup UI + self._default_size_applied = False self.setup_ui() + # Connect automatic change detection (BaseManagedWindow feature) + # This automatically calls detect_changes() when any parameter changes + self._connect_change_detection() + + # Initialize save button state + self.detect_changes() + logger.debug(f"Config window initialized for {config_class.__name__}") def _update_window_title_dirty_marker(self) -> None: @@ -176,83 +343,135 @@ def _update_window_title_dirty_marker(self) -> None: elif not is_dirty and has_marker: self.setWindowTitle(self._base_window_title) - # Update header label with both asterisk and underline - if hasattr(self, '_header_label'): - header_text = f"Configure {self.config_class.__name__}" - if is_dirty: - self._header_label.setText(f"* {header_text}") - else: - self._header_label.setText(header_text) + # Update header label with both asterisk and underline (matches DualEditorWindow) + if hasattr(self, "_header_label"): + header_text = ( + f"{'* ' if is_dirty else ''}Configure {self.config_class.__name__}" + ) + self._header_label.setText(header_text) + # Apply underline for signature diff (independent of dirty) font = self._header_label.font() font.setUnderline(has_sig_diff) self._header_label.setFont(font) + def detect_changes(self): + """Detect if changes have been made using ObjectState's dirty tracking. + + This replaces old snapshot-based approach with ObjectState's built-in + dirty tracking, which automatically detects changes to any parameter + (including nested fields) by comparing current values to saved baseline. + """ + # Use ObjectState's dirty tracking instead of custom snapshot comparison + has_changes = bool(self.state.is_raw_dirty) if self.state else False + + if has_changes != self.has_changes: + self.has_changes = has_changes + self.changes_detected.emit(has_changes) + + def on_changes_detected(self, has_changes: bool): + """Handle changes detection.""" + # Enable/disable save button based on changes + self._save_button.setEnabled(has_changes) + def setup_ui(self): """Setup the user interface.""" self.setWindowTitle(self._base_window_title) self.setModal(False) # Non-modal like plate manager and pipeline editor - self.setMinimumSize(600, 400) - self.resize(800, 600) - - layout = QVBoxLayout(self) - layout.setContentsMargins(4, 4, 4, 4) - layout.setSpacing(4) - - # Header with title, help button, and action buttons - header_widget = QWidget() - header_layout = QHBoxLayout(header_widget) - header_layout.setContentsMargins(4, 2, 4, 2) - - self._header_label = QLabel(f"Configure {self.config_class.__name__}") - self._header_label.setFont(QFont("Arial", 14, QFont.Weight.Bold)) - self._header_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};") - header_layout.addWidget(self._header_label) - - # Add help button for the dataclass itself - self._help_btn = None - if dataclasses.is_dataclass(self.config_class): - self._help_btn = HelpButton(help_target=self.config_class, text="Help", color_scheme=self.color_scheme) - self._help_btn.setMaximumWidth(80) - header_layout.addWidget(self._help_btn) + if self.size().isEmpty(): + self.resize(550, 600) - header_layout.addStretch() + self._layout = QVBoxLayout(self) + self._layout.setContentsMargins(4, 4, 4, 4) + self._layout.setSpacing(4) - # Add action buttons to header (top right) + # Responsive header layout with staged wrapping (content-based) + # All start on same row: Title | Reset | View Code | Help | Cancel | Save button_styles = self.style_generator.generate_config_button_styles() + title_text = f"Configure {self.config_class.__name__}" + self._header_label = QLabel(title_text) + self._header_label.setFont(QFont("Arial", 14, QFont.Weight.Bold)) + self._header_label.setWordWrap(True) + self._header_label.setMinimumWidth(0) + self._header_label.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed + ) + self._header_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};" + ) + title_group = QWidget() + title_layout = QHBoxLayout(title_group) + title_layout.setContentsMargins(0, 0, 0, 0) + title_layout.setSpacing(4) + title_layout.addWidget(self._header_label) + + reset_button = QPushButton("Reset to Defaults") + reset_button.setFixedHeight(CURRENT_LAYOUT.button_height) + reset_button.setMinimumWidth(100) + reset_button.clicked.connect(self.reset_to_defaults) + reset_button.setStyleSheet(button_styles["compact"]) - # View Code button view_code_button = QPushButton("View Code") - view_code_button.setFixedHeight(28) + view_code_button.setFixedHeight(CURRENT_LAYOUT.button_height) view_code_button.setMinimumWidth(80) view_code_button.clicked.connect(self._view_code) - view_code_button.setStyleSheet(button_styles["reset"]) - header_layout.addWidget(view_code_button) + view_code_button.setStyleSheet(button_styles["compact"]) - # Reset button - reset_button = QPushButton("Reset to Defaults") - reset_button.setFixedHeight(28) - reset_button.setMinimumWidth(100) - reset_button.clicked.connect(self.reset_to_defaults) - reset_button.setStyleSheet(button_styles["reset"]) - header_layout.addWidget(reset_button) + group_reset = QWidget() + group_reset_layout = QHBoxLayout(group_reset) + group_reset_layout.setContentsMargins(0, 0, 0, 0) + group_reset_layout.setSpacing(4) + group_reset_layout.addWidget(reset_button) + group_reset_layout.addWidget(view_code_button) + + self._help_btn = None + group_help = QWidget() + group_help_layout = QHBoxLayout(group_help) + group_help_layout.setContentsMargins(0, 0, 0, 0) + group_help_layout.setSpacing(4) + group_help_layout.setAlignment(Qt.AlignmentFlag.AlignLeft) + if dataclasses.is_dataclass(self.config_class): + self._help_btn = HelpButton( + help_target=self.config_class, + text="Help", + color_scheme=self.color_scheme, + scope_accent_color=getattr(self, "_scope_accent_color", None), + ) + self._help_btn.setMaximumWidth(80) + self._help_btn.setFixedHeight(CURRENT_LAYOUT.button_height) + group_help_layout.addWidget(self._help_btn) - # Cancel button cancel_button = QPushButton("Cancel") - cancel_button.setFixedHeight(28) + cancel_button.setFixedHeight(CURRENT_LAYOUT.button_height) cancel_button.setMinimumWidth(70) cancel_button.clicked.connect(self.reject) - cancel_button.setStyleSheet(button_styles["cancel"]) - header_layout.addWidget(cancel_button) + cancel_button.setStyleSheet(button_styles["compact"]) - # Save button self._save_button = QPushButton("Save") - self._save_button.setFixedHeight(28) + self._save_button.setFixedHeight(CURRENT_LAYOUT.button_height) self._save_button.setMinimumWidth(70) self._setup_save_button(self._save_button, self.save_config) - self._save_button.setStyleSheet(button_styles["save"]) - header_layout.addWidget(self._save_button) + self._save_button.setStyleSheet(button_styles["compact"]) + + group_save = QWidget() + group_save_layout = QHBoxLayout(group_save) + group_save_layout.setContentsMargins(0, 0, 0, 0) + group_save_layout.setSpacing(4) + group_save_layout.addWidget(cancel_button) + group_save_layout.addWidget(self._save_button) + + header_widget = StagedWrapLayout(parent=self) + header_widget.set_groups( + [ + ("title", title_group), + ("group_help", group_help), + ("group_reset", group_reset), + ("group_save", group_save), + ], + ["title", "group_save", "group_help", "group_reset"], + right_align_names=["group_save"], + ) - layout.addWidget(header_widget) + self._layout.addWidget(header_widget) # Create splitter with tree view on left and form on right self.splitter = QSplitter(Qt.Orientation.Horizontal) @@ -267,8 +486,12 @@ def setup_ui(self): # Always use scroll area for consistent navigation behavior self.scroll_area = QScrollArea() self.scroll_area.setWidgetResizable(True) - self.scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) - self.scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff) + self.scroll_area.setVerticalScrollBarPolicy( + Qt.ScrollBarPolicy.ScrollBarAsNeeded + ) + self.scroll_area.setHorizontalScrollBarPolicy( + Qt.ScrollBarPolicy.ScrollBarAlwaysOff + ) self.scroll_area.setWidget(self.form_manager) self.splitter.addWidget(self.scroll_area) @@ -276,16 +499,49 @@ def setup_ui(self): self.splitter.setSizes([300, 700]) # Install collapsible splitter helper for double-click toggle - self.splitter_helper = CollapsibleSplitterHelper(self.splitter, left_panel_index=0) + self.splitter_helper = CollapsibleSplitterHelper( + self.splitter, left_panel_index=0 + ) self.splitter_helper.set_initial_size(300) # Add splitter with stretch factor so it expands to fill available space - layout.addWidget(self.splitter, 1) # stretch factor = 1 + self._layout.addWidget(self.splitter, 1) # stretch factor = 1 # Apply centralized styling (config window style includes tree styling now) self.setStyleSheet( - self.style_generator.generate_config_window_style() + "\n" + - self.style_generator.generate_tree_widget_style() + self.style_generator.generate_config_window_style() + + "\n" + + self.style_generator.generate_tree_widget_style() + ) + + # CRITICAL: Initialize scope-based border styling AFTER widgets are created + # This ensures widgets exist when _apply_scope_accent_styling() tries to style them + # (mirrors DualEditorWindow pattern which calls _init_scope_border in setup_connections) + if self.scope_id: + self._init_scope_border() + + def showEvent(self, a0) -> None: + super().showEvent(a0) + if not getattr(self, "_default_size_applied", False): + self.resize(550, 600) + QTimer.singleShot(0, lambda: self.resize(550, 600)) + self._default_size_applied = True + self.setProperty("_fixed_default_size", True) + self._log_window_size("shown") + + def resizeEvent(self, a0) -> None: + super().resizeEvent(a0) + self._log_window_size("resized") + + def _log_window_size(self, context: str) -> None: + size = self.size() + logger.debug( + "Config window %s size=%dx%d pos=%d,%d", + context, + size.width(), + size.height(), + self.x(), + self.y(), ) def _apply_scope_accent_styling(self) -> None: @@ -302,27 +558,36 @@ def _apply_scope_accent_styling(self) -> None: hex_color = accent_color.name() - # Style Save button directly + # Style Save button with hover effect save_button_style = f""" - background-color: {hex_color}; - color: white; - border: none; - border-radius: 3px; - padding: 8px; + QPushButton {{ + background-color: {hex_color}; + color: white; + border: none; + border-radius: 3px; + padding: 8px; + }} + QPushButton:hover {{ + background-color: {accent_color.lighter(115).name()}; + }} """ - if hasattr(self, '_save_button'): + if hasattr(self, "_save_button"): self._save_button.setStyleSheet(save_button_style) # Style header label with scope accent color - if hasattr(self, '_header_label'): + if hasattr(self, "_header_label"): self._header_label.setStyleSheet(f"color: {hex_color};") # Style tree selection with scope accent tree_style = self.get_scope_tree_selection_stylesheet() - if tree_style and hasattr(self, 'tree_widget'): + if tree_style and hasattr(self, "tree_widget"): current_style = self.tree_widget.styleSheet() or "" self.tree_widget.setStyleSheet(f"{current_style}\n{tree_style}") + # Style help button with scope accent color + if hasattr(self, "_help_btn") and self._help_btn: + self._help_btn.set_scope_accent_color(accent_color) + def _create_inheritance_tree(self) -> QTreeWidget: """Create tree widget showing inheritance hierarchy for navigation.""" # Pass form_manager as flash_manager - tree reads from SAME _flash_colors dict as groupboxes @@ -330,7 +595,8 @@ def _create_inheritance_tree(self) -> QTreeWidget: # Pass state for automatic dirty tracking subscription (handled by helper) tree = self.tree_helper.create_tree_widget( flash_manager=self.form_manager, - state=self.state + state=self.state, + strip_config_suffix=True, ) self.tree_helper.populate_from_root_dataclass(tree, self.config_class) @@ -352,34 +618,42 @@ def _on_tree_item_double_clicked(self, item: QTreeWidgetItem, column: int): return # Check if this item is ui_hidden - if so, ignore the double-click - if data.get('ui_hidden', False): + if data.get("ui_hidden", False): logger.debug("Ignoring double-click on ui_hidden item") return - item_type = data.get('type') + item_type = data.get("type") - if item_type == 'dataclass': + if item_type == "dataclass": # Navigate to the dataclass section in the form - field_name = data.get('field_name') - if field_name: - self._scroll_to_section(field_name) - logger.debug(f"Navigating to section: {field_name}") + field_path = data.get("field_path") or data.get("field_name") + if field_path: + self._scroll_to_section(field_path) + logger.debug(f"Navigating to section: {field_path}") else: - class_obj = data.get('class') - class_name = getattr(class_obj, '__name__', 'Unknown') if class_obj else 'Unknown' + class_obj = data.get("class") + class_name = ( + getattr(class_obj, "__name__", "Unknown") + if class_obj + else "Unknown" + ) logger.debug(f"Double-clicked on root dataclass: {class_name}") - elif item_type == 'inheritance_link': + elif item_type == "inheritance_link": # Navigate to the parent class section in the form - target_class = data.get('target_class') + target_class = data.get("target_class") if target_class: # Find the field that has this type (or its lazy version) field_name = self._find_field_for_class(target_class) if field_name: self._scroll_to_section(field_name) - logger.debug(f"Navigating to inherited section: {field_name} (class: {target_class.__name__})") + logger.debug( + f"Navigating to inherited section: {field_name} (class: {target_class.__name__})" + ) else: - logger.warning(f"Could not find field for class {target_class.__name__}") + logger.warning( + f"Could not find field for class {target_class.__name__}" + ) def _find_field_for_class(self, target_class) -> str: """Find the field name that has the given class type (or its lazy version).""" @@ -410,23 +684,17 @@ def _find_field_for_class(self, target_class) -> str: # _scroll_to_section is provided by ScrollableFormMixin - - - - - - def update_widget_value(self, widget: QWidget, value: Any): """ Update widget value without triggering signals. - + Args: widget: Widget to update value: New value """ # Temporarily block signals to avoid recursion widget.blockSignals(True) - + try: if isinstance(widget, QCheckBox): widget.setChecked(bool(value)) @@ -443,7 +711,7 @@ def update_widget_value(self, widget: QWidget, value: Any): widget.setText(str(value) if value is not None else "") finally: widget.blockSignals(False) - + def reset_to_defaults(self): """Reset all parameters using centralized service with full sophistication.""" # Service layer now contains ALL the sophisticated logic previously in infrastructure classes @@ -465,15 +733,21 @@ def save_config(self, *, close_window=True): # CRITICAL: Set flag to prevent refresh_config from recreating the form # The window already has the correct data - it just saved it! self._saving = True - logger.info(f"🔍 SAVE_CONFIG: Set _saving=True before callback (id={id(self)})") + logger.info( + f"🔍 SAVE_CONFIG: Set _saving=True before callback (id={id(self)})" + ) try: # Emit signal and call callback self.config_saved.emit(new_config) if self.on_save_callback: - logger.info(f"🔍 SAVE_CONFIG: Calling on_save_callback (id={id(self)})") + logger.info( + f"🔍 SAVE_CONFIG: Calling on_save_callback (id={id(self)})" + ) self.on_save_callback(new_config) - logger.info(f"🔍 SAVE_CONFIG: Returned from on_save_callback (id={id(self)})") + logger.info( + f"🔍 SAVE_CONFIG: Returned from on_save_callback (id={id(self)})" + ) finally: self._saving = False logger.info(f"🔍 SAVE_CONFIG: Reset _saving=False (id={id(self)})") @@ -483,13 +757,17 @@ def save_config(self, *, close_window=True): # Also update LIVE thread-local to match saved set_saved_global_config(self.config_class, new_config) set_live_global_config(self.config_class, new_config) - logger.debug(f"Updated SAVED and LIVE thread-local {self.config_class.__name__} on SAVE") + logger.debug( + f"Updated SAVED and LIVE thread-local {self.config_class.__name__} on SAVE" + ) # CRITICAL: Invalidate ALL descendant caches so they re-resolve with the new SAVED thread-local # This is necessary when saving None values - descendants must pick up the new None # instead of continuing to use cached values resolved from the old saved thread-local ObjectStateRegistry.increment_token(notify=True) - logger.debug(f"Invalidated all descendant caches after updating SAVED thread-local") + logger.debug( + f"Invalidated all descendant caches after updating SAVED thread-local" + ) self._original_global_config_snapshot = copy.deepcopy(new_config) self._global_context_dirty = False @@ -502,8 +780,9 @@ def save_config(self, *, close_window=True): except Exception as e: logger.error(f"Failed to save configuration: {e}") - QMessageBox.critical(self, "Save Error", f"Failed to save configuration:\n{e}") - + QMessageBox.critical( + self, "Save Error", f"Failed to save configuration:\n{e}" + ) def _view_code(self): """Open code editor to view/edit the configuration as Python code.""" @@ -526,15 +805,17 @@ def _view_code(self): # Launch editor editor_service = SimpleCodeEditorService(self) - use_external = os.environ.get('OPENHCS_USE_EXTERNAL_EDITOR', '').lower() in ('1', 'true', 'yes') + use_external = os.environ.get( + "OPENHCS_USE_EXTERNAL_EDITOR", "" + ).lower() in ("1", "true", "yes") editor_service.edit_code( initial_content=python_code, title=f"View/Edit {self.config_class.__name__}", callback=self._handle_edited_config_code, use_external=use_external, - code_type='config', - code_data={'config_class': self.config_class, 'clean_mode': True} + code_type="config", + code_data={"config_class": self.config_class, "clean_mode": True}, ) except Exception as e: @@ -552,12 +833,14 @@ def _handle_edited_config_code(self, edited_code: str): with CodeEditorFormUpdater.patch_lazy_constructors(): exec(edited_code, namespace) - new_config = namespace.get('config') + new_config = namespace.get("config") if not new_config: raise ValueError("No 'config' variable found in edited code") if not isinstance(new_config, self.config_class): - raise ValueError(f"Expected {self.config_class.__name__}, got {type(new_config).__name__}") + raise ValueError( + f"Expected {self.config_class.__name__}, got {type(new_config).__name__}" + ) # Update current config self.current_config = new_config @@ -575,7 +858,9 @@ def _handle_edited_config_code(self, edited_code: str): if is_global_config_type(self.config_class): # For global configs: Update thread-local context immediately set_global_config_for_editing(self.config_class, new_config) - logger.debug(f"Updated thread-local {self.config_class.__name__} context") + logger.debug( + f"Updated thread-local {self.config_class.__name__} context" + ) self._global_context_dirty = True # For PipelineConfig: No context update needed here # The orchestrator.apply_pipeline_config() happens in the save callback @@ -592,7 +877,9 @@ def _handle_edited_config_code(self, edited_code: str): except Exception as e: logger.error(f"Failed to apply edited config code: {e}") - QMessageBox.critical(self, "Code Edit Error", f"Failed to apply edited code:\n{e}") + QMessageBox.critical( + self, "Code Edit Error", f"Failed to apply edited code:\n{e}" + ) def _on_global_config_field_changed(self, param_name: str, value: Any): """Track that global config has unsaved changes. @@ -614,31 +901,42 @@ def _update_form_from_config(self, new_config): CodeEditorFormUpdater.update_form_from_instance( self.form_manager, new_config, - broadcast_callback=self._broadcast_config_changed, ) def reject(self): - """Handle dialog rejection (Cancel button).""" - if (is_global_config_type(self.config_class) and - getattr(self, '_global_context_dirty', False) and - self._original_global_config_snapshot is not None): - set_global_config_for_editing(self.config_class, - copy.deepcopy(self._original_global_config_snapshot)) + """Handle dialog rejection (Cancel button). + + Restores global config context and ObjectState to last saved state. + """ + # Restore global config context if dirty + if ( + is_global_config_type(self.config_class) + and getattr(self, "_global_context_dirty", False) + and self._original_global_config_snapshot is not None + ): + set_global_config_for_editing( + self.config_class, copy.deepcopy(self._original_global_config_snapshot) + ) self._global_context_dirty = False logger.debug(f"Restored {self.config_class.__name__} context after cancel") self.config_cancelled.emit() - super().reject() # BaseFormDialog handles unregistration + + # CRITICAL: super().reject() calls state.restore_saved() to undo ALL unsaved changes + # This restores all parameters (not just global context) to last saved state + super().reject() # BaseFormDialog handles state restoration + unregistration # CRITICAL: Trigger global refresh AFTER unregistration so other windows # re-collect live context without this cancelled window's values # This ensures group_by selector and other placeholders sync correctly ObjectStateRegistry.increment_token() - logger.debug(f"Triggered global refresh after cancelling {self.config_class.__name__} editor") + logger.debug( + f"Triggered global refresh after cancelling {self.config_class.__name__} editor" + ) def _get_form_managers(self): """Return list of form managers to unregister (required by BaseFormDialog).""" - if hasattr(self, 'form_manager'): + if hasattr(self, "form_manager"): return [self.form_manager] return [] diff --git a/openhcs/pyqt_gui/windows/dual_editor_window.py b/openhcs/pyqt_gui/windows/dual_editor_window.py index 38d6336ba..af22ff9cf 100644 --- a/openhcs/pyqt_gui/windows/dual_editor_window.py +++ b/openhcs/pyqt_gui/windows/dual_editor_window.py @@ -8,14 +8,22 @@ import logging from dataclasses import fields, is_dataclass from pathlib import Path -from typing import Optional, Callable, Dict, List +from typing import Optional, Callable, Dict, List, Any from PyQt6.QtWidgets import ( - QDialog, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, - QTabWidget, QWidget, QStackedWidget, QSizePolicy, QMessageBox + QDialog, + QVBoxLayout, + QHBoxLayout, + QPushButton, + QLabel, + QTabWidget, + QWidget, + QStackedWidget, + QSizePolicy, + QMessageBox, ) from PyQt6.QtCore import pyqtSignal, Qt, QTimer -from PyQt6.QtGui import QFont +from PyQt6.QtGui import QFont, QShowEvent from openhcs.core.steps.function_step import FunctionStep from openhcs.constants.constants import GroupBy @@ -29,11 +37,14 @@ from pyqt_reactive.theming import ColorScheme from pyqt_reactive.theming import StyleSheetGenerator from pyqt_reactive.services.scope_token_service import ScopeTokenService +from pyqt_reactive.services.function_navigation import is_function_field_path from pyqt_reactive.widgets.function_list_editor import FunctionListEditorWidget -from openhcs.pyqt_gui.windows.base_form_dialog import BaseFormDialog +from pyqt_reactive.forms.layout_constants import CURRENT_LAYOUT +from pyqt_reactive.widgets.shared import BaseFormDialog, ResponsiveTwoRowWidget from openhcs.config_framework.object_state import ObjectStateRegistry from openhcs.introspection import UnifiedParameterAnalyzer from openhcs.pyqt_gui.widgets.step_parameter_editor import StepParameterEditorWidget + logger = logging.getLogger(__name__) @@ -53,10 +64,18 @@ class DualEditorWindow(BaseFormDialog): step_saved = pyqtSignal(object) # FunctionStep step_cancelled = pyqtSignal() changes_detected = pyqtSignal(bool) # has_changes - - def __init__(self, step_data: Optional[FunctionStep] = None, is_new: bool = False, - on_save_callback: Optional[Callable] = None, color_scheme: Optional[ColorScheme] = None, - orchestrator=None, gui_config=None, parent=None, step_index: Optional[int] = None): + + def __init__( + self, + step_data: Optional[FunctionStep] = None, + is_new: bool = False, + on_save_callback: Optional[Callable] = None, + color_scheme: Optional[ColorScheme] = None, + orchestrator=None, + gui_config=None, + parent=None, + step_index: Optional[int] = None, + ): """ Initialize the dual editor window. @@ -87,7 +106,7 @@ def __init__(self, step_data: Optional[FunctionStep] = None, is_new: bool = Fals self.is_new = is_new self.on_save_callback = on_save_callback self.orchestrator = orchestrator # Store orchestrator for context management - + # Pattern management (extracted from Textual version) self.pattern_manager = PatternDataManager() @@ -103,24 +122,83 @@ def __init__(self, step_data: Optional[FunctionStep] = None, is_new: bool = Fals self.editing_step = self._create_new_step() self.original_step = None - # Snapshot of last-saved state for change detection - self._baseline_snapshot = self._serialize_for_change_detection(self.editing_step) - - # Change tracking + # Change tracking (now uses ObjectState.dirty_fields instead of snapshots) self.has_changes = False self.current_tab = "step" - + + # Change detection callback (ObjectState.on_state_changed expects a no-arg callable) + self._dirty_detect_callback = lambda: self.detect_changes() + # UI components self.tab_widget: Optional[QTabWidget] = None - self.parameter_editors: Dict[str, QWidget] = {} # Map tab titles to editor widgets + self.parameter_editors: Dict[ + str, QWidget + ] = {} # Map tab titles to editor widgets self.class_hierarchy: List = [] # Store inheritance hierarchy info - + + # Editors are created during setup_ui(); initialize here so scope styling + # hooks can run during _init_scope_border() without attribute errors. + self.step_editor = None + self.func_editor = None + + self._flash_overlay = None # Window flash overlay for visual feedback + self._flash_overlay_cleaned = False # Track if overlay was cleaned up + self._step_buttons_widget: Optional[QWidget] = ( + None # Button widget from step editor + ) + self._func_buttons_widget: Optional[QWidget] = ( + None # Button widget from func editor + ) + # Setup UI self.setup_ui() self.setup_connections() - + + # Ensure the initial save button state reflects current ObjectState. + # setup_ui() may call detect_changes() before changes_detected is connected. + self.detect_changes() + + # Connect automatic change detection (BaseManagedWindow feature) + # This automatically calls detect_changes() when any parameter changes + self._connect_change_detection() + logger.debug(f"Dual editor window initialized (new={is_new})") + def showEvent(self, event: QShowEvent) -> None: + super().showEvent(event) + if not getattr(self, "_default_size_applied", False): + self.resize(550, 600) + self._default_size_applied = True + self._log_window_size("shown") + + def resizeEvent(self, event) -> None: + super().resizeEvent(event) + self._log_window_size("resized") + + def _log_window_size(self, context: str) -> None: + size = self.size() + logger.debug( + "Dual editor window %s size=%dx%d pos=%d,%d", + context, + size.width(), + size.height(), + self.x(), + self.y(), + ) + + @property + def state(self): + """Expose step_editor's ObjectState for BaseManagedWindow compatibility. + + This allows BaseManagedWindow.reject() to find and restore the state + when the window is cancelled or closed without saving. + + Returns None if step_editor hasn't been created yet. + """ + if self.step_editor: + return self.step_editor.state + return None + def set_original_step_for_change_detection(self): """Set the original step for change detection. Must be called within proper context.""" # Original step is already set in __init__ when working on a copy @@ -129,78 +207,96 @@ def set_original_step_for_change_detection(self): def setup_ui(self): """Setup the user interface.""" - self._update_window_title() - self.resize(700, 500) + # Note: _update_window_title() is called at the end after header_label is created + if self.size().isEmpty(): + self.resize(550, 600) layout = QVBoxLayout(self) layout.setSpacing(5) layout.setContentsMargins(5, 5, 5, 5) - # Single row: tabs + title + status + buttons - tab_row = QHBoxLayout() - tab_row.setContentsMargins(5, 5, 5, 5) - tab_row.setSpacing(10) + # Get centralized button styles + button_styles = self.style_generator.generate_config_button_styles() - # Tab widget (tabs on the left) - self.tab_widget = QTabWidget() - self.tab_bar = self.tab_widget.tabBar() - self.tab_bar.setExpanding(False) - self.tab_bar.setUsesScrollButtons(False) - tab_row.addWidget(self.tab_bar, 0) + # Title row: title on left, buttons on right (responsive - wraps to row 2 when narrow) + self._title_header = ResponsiveTwoRowWidget(width_threshold=400, parent=self) + self._title_header.setStyleSheet(f""" + QWidget {{ + background-color: {self.color_scheme.to_hex(self.color_scheme.panel_bg)}; + border-radius: 3px; + }} + """) - # Title label + # Title label (left side - stays in row 1) self.header_label = QLabel() self.header_label.setFont(QFont("Arial", 14, QFont.Weight.Bold)) - self.header_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};") - self.header_label.setSizePolicy(QSizePolicy.Policy.Ignored, QSizePolicy.Policy.Preferred) - tab_row.addWidget(self.header_label, 1) - - tab_row.addStretch() - - # Status indicator - self.changes_label = QLabel("") - self.changes_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.status_warning)}; font-style: italic;") - tab_row.addWidget(self.changes_label) - - # Get centralized button styles - button_styles = self.style_generator.generate_config_button_styles() + self.header_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; background-color: transparent;" + ) + self._title_header.add_left_widget(self.header_label) - # Cancel button + # Cancel button (right side - wraps to row 2 when narrow) cancel_button = QPushButton("Cancel") - cancel_button.setFixedHeight(28) + cancel_button.setFixedHeight(CURRENT_LAYOUT.button_height) cancel_button.setMinimumWidth(70) + cancel_button.setSizePolicy( + QSizePolicy.Policy.Maximum, QSizePolicy.Policy.Fixed + ) cancel_button.clicked.connect(self.cancel_edit) - cancel_button.setStyleSheet(button_styles["cancel"]) - tab_row.addWidget(cancel_button) + cancel_button.setStyleSheet(button_styles["compact"]) + self._title_header.add_right_widget(cancel_button) - # Save/Create button + # Save/Create button (right side - wraps to row 2 when narrow) self.save_button = QPushButton() self._update_save_button_text() - self.save_button.setFixedHeight(28) + self.save_button.setFixedHeight(CURRENT_LAYOUT.button_height) self.save_button.setMinimumWidth(70) + self.save_button.setSizePolicy( + QSizePolicy.Policy.Maximum, QSizePolicy.Policy.Fixed + ) self.save_button.setEnabled(False) self._setup_save_button(self.save_button, self.save_edit) - self.save_button.setStyleSheet(button_styles["save"] + f""" + self._save_button_base_style = ( + button_styles["compact"] + + f""" QPushButton:disabled {{ background-color: {self.color_scheme.to_hex(self.color_scheme.panel_bg)}; color: {self.color_scheme.to_hex(self.color_scheme.border_light)}; border: none; }} - """) - tab_row.addWidget(self.save_button) + """ + ) + self.save_button.setStyleSheet(self._save_button_base_style) + self._title_header.add_right_widget(self.save_button) + + # Connect change detection BEFORE tabs are created. + # create_step_tab() can call detect_changes() during setup. + self.changes_detected.connect(self.on_changes_detected) + + layout.addWidget(self._title_header) + + # Tab row: tabs on left, action buttons on right (responsive - buttons wrap when narrow) + self._tab_row = ResponsiveTwoRowWidget(width_threshold=600, parent=self) - layout.addLayout(tab_row) + # Tab widget (left side - stays in row 1) + self.tab_widget = QTabWidget() + self.tab_bar = self.tab_widget.tabBar() + self.tab_bar.setExpanding(False) + self.tab_bar.setUsesScrollButtons(False) + self.tab_bar.setFixedHeight(CURRENT_LAYOUT.button_height) + self._tab_row.add_left_widget(self.tab_bar) # Style the tab bar self.tab_bar.setStyleSheet(f""" QTabBar::tab {{ background-color: {self.color_scheme.to_hex(self.color_scheme.input_bg)}; color: white; - padding: 8px 16px; + padding: 0px 16px; margin-right: 2px; border-top-left-radius: 4px; border-top-right-radius: 4px; border: none; + height: {CURRENT_LAYOUT.button_height}px; }} QTabBar::tab:selected {{ background-color: {self.color_scheme.to_hex(self.color_scheme.selection_bg)}; @@ -210,10 +306,39 @@ def setup_ui(self): }} """) + # Container to hold the action buttons from all tabs + self._active_buttons_container = QWidget() + self._active_buttons_layout = QHBoxLayout(self._active_buttons_container) + self._active_buttons_layout.setContentsMargins(0, 0, 0, 0) + self._active_buttons_layout.setSpacing(0) + self._tab_row.add_right_widget(self._active_buttons_container) + + layout.addWidget(self._tab_row) + + # Scope ID for singleton behavior and border styling. + # Must be initialized BEFORE creating editors so scope accent color is available. + if self.orchestrator is None: + raise RuntimeError( + "DualEditorWindow requires orchestrator to build scope styling" + ) + if self.editing_step is None: + raise RuntimeError( + "DualEditorWindow requires editing_step to build scope styling" + ) + self.scope_id = self._build_step_scope_id() + logger.debug( + "[DUAL_EDITOR] Set scope_id to: %s, calling _init_scope_border()", + self.scope_id, + ) + self._init_scope_border() + # Create tabs (this adds content to the tab widget) self.create_step_tab() self.create_function_tab() + # Editors now exist; apply scope styling to their widget trees. + self._apply_scope_accent_styling() + # Add the tab widget's content area (stacked widget) below the tab row # The tab bar is already in tab_row, so we only add the content pane here content_container = QWidget() @@ -228,8 +353,11 @@ def setup_ui(self): layout.addWidget(content_container) - # Apply centralized styling - self.setStyleSheet(self.style_generator.generate_config_window_style()) + # Connect tab change to swap buttons + self.tab_widget.currentChanged.connect(self._on_tab_changed) + + # Setup tab button containers (extract buttons from editors once) + QTimer.singleShot(0, self._setup_tab_button_containers) # Debounce timer for function editor synchronization (batches rapid updates) self._function_sync_timer = QTimer(self) @@ -237,9 +365,53 @@ def setup_ui(self): self._function_sync_timer.timeout.connect(self._flush_function_editor_sync) self._pending_function_editor_sync = False - # Scope ID for singleton behavior and border styling - if getattr(self, "orchestrator", None) and getattr(self, "editing_step", None): - self.scope_id = self._build_step_scope_id() + # Update title now that header_label exists + self._update_window_title() + + def _setup_tab_button_containers(self) -> None: + """Extract buttons from editors and add to active buttons container. + + This is called ONCE after tabs are created to avoid adding widgets + to the layout multiple times. + """ + # Extract step editor buttons widget and add to layout + self._step_buttons_widget = self.step_editor.get_action_buttons() + if self._step_buttons_widget: + self._step_buttons_widget.setVisible(False) + self._active_buttons_layout.addWidget(self._step_buttons_widget) + + # Extract function editor buttons widget and add to layout + self._func_buttons_widget = self.func_editor.get_action_buttons() + if self._func_buttons_widget: + self._func_buttons_widget.setVisible(False) + self._active_buttons_layout.addWidget(self._func_buttons_widget) + + # Show buttons for the initially selected tab + self._show_tab_buttons() + + def _show_tab_buttons(self) -> None: + """Show only buttons for the currently active tab. + + Toggles visibility of pre-embedded button widgets based on + which tab is currently selected. + """ + current_index = self.tab_widget.currentIndex() + + # Hide all button widgets first + if self._step_buttons_widget: + self._step_buttons_widget.setVisible(False) + if self._func_buttons_widget: + self._func_buttons_widget.setVisible(False) + + # Show only the active tab's buttons + if current_index == 0 and self._step_buttons_widget: + self._step_buttons_widget.setVisible(True) + elif current_index == 1 and self._func_buttons_widget: + self._func_buttons_widget.setVisible(True) + + def _on_tab_changed(self, index: int) -> None: + """Handle tab change - show appropriate action buttons.""" + self._show_tab_buttons() def _update_window_title(self): """Update window title with dirty marker and signature diff underline. @@ -247,18 +419,39 @@ def _update_window_title(self): Two orthogonal visual semantics: - Asterisk (*): dirty (resolved_live != resolved_saved) - Underline: signature diff (raw != signature default) + + Reads step name from ObjectState for live updates when user edits the name field. """ - base_title = "New Step" if self.is_new else f"Edit Step: {self.editing_step.name}" + # Get step name from ObjectState if available, otherwise fall back to editing_step + step_editor = getattr(self, "step_editor", None) + if step_editor and step_editor.state: + # Read name from ObjectState (updates live as user types) + current_values = step_editor.state.get_current_values() + step_name = current_values.get("name", "Unnamed") + else: + # Fallback to editing_step.name during initial setup + step_name = ( + getattr(self.editing_step, "name", "Unnamed") + if self.editing_step + else "Unnamed" + ) + + base_title = f"{'New' if self.is_new else 'Edit'} Step: {step_name}" self._base_window_title = base_title - # step_editor doesn't exist yet during initial setup_ui() call - step_editor = getattr(self, 'step_editor', None) - is_dirty = bool(step_editor and step_editor.state.dirty_fields) - has_sig_diff = bool(step_editor and step_editor.state.signature_diff_fields) + # Check dirty state from ObjectState + is_dirty = bool( + step_editor and step_editor.state and step_editor.state.dirty_fields + ) + has_sig_diff = bool( + step_editor + and step_editor.state + and step_editor.state.signature_diff_fields + ) title = f"* {base_title}" if is_dirty else base_title self.setWindowTitle(title) - if getattr(self, 'header_label', None): + if getattr(self, "header_label", None): self.header_label.setText(title) # Apply underline for signature diff (independent of dirty) font = self.header_label.font() @@ -266,86 +459,127 @@ def _update_window_title(self): self.header_label.setFont(font) def _update_save_button_text(self): - if hasattr(self, 'save_button'): - new_text = "Create" if getattr(self, 'is_new', False) else "Save" - logger.info(f"🔘 Updating save button text: is_new={self.is_new} → '{new_text}'") - self.save_button.setText(new_text) + base_text = "Create" if self.is_new else "Save" + # Add asterisk if there are unsaved changes + has_changes = self.has_changes + new_text = f"* {base_text}" if has_changes else base_text + logger.info( + f"🔘 Updating save button text: is_new={self.is_new}, has_changes={has_changes} → '{new_text}'" + ) + self.save_button.setText(new_text) + if getattr(self, "_save_button_base_style", ""): + self.save_button.setStyleSheet(self._save_button_base_style) def _apply_scope_accent_styling(self) -> None: - """Apply scope accent color to DualEditorWindow-specific elements.""" - super()._apply_scope_accent_styling() + """Apply scope accent color to dual editor window elements. + Overrides the empty implementation in ScopedBorderMixin to style: + - Save button + - Tab bar tabs + - Window flash overlay + """ accent_color = self.get_scope_accent_color() - if not accent_color: - return + if accent_color is None: + raise RuntimeError( + "Scope accent color is missing; call _init_scope_border() after setting scope_id" + ) + + # Store for child widgets that need the computed accent. + self._scope_accent_color = accent_color hex_color = accent_color.name() - hex_lighter = accent_color.lighter(115).name() - hex_darker = accent_color.darker(115).name() - # Style Save button (preserve disabled state styling) - if hasattr(self, 'save_button'): - self.save_button.setStyleSheet(f""" - QPushButton {{ - background-color: {hex_color}; - color: white; - border: none; - border-radius: 3px; - padding: 8px; - }} - QPushButton:disabled {{ - background-color: {self.color_scheme.to_hex(self.color_scheme.panel_bg)}; - color: {self.color_scheme.to_hex(self.color_scheme.border_light)}; - border: none; - }} - """) + # Style Save button with scope accent color + self._save_button_base_style = f""" + QPushButton {{ + background-color: {hex_color}; + color: white; + border: none; + border-radius: 3px; + padding: 8px; + }} + QPushButton:hover {{ + background-color: {accent_color.lighter(115).name()}; + }} + QPushButton:disabled {{ + background-color: {self.color_scheme.to_hex(self.color_scheme.panel_bg)}; + color: {self.color_scheme.to_hex(self.color_scheme.border_light)}; + border: none; + }} + """ + self.save_button.setStyleSheet(self._save_button_base_style) - # Style header label if present - if hasattr(self, 'header_label'): - self.header_label.setStyleSheet(f"color: {hex_color};") + # Style header label with scope accent color + self.header_label.setStyleSheet(f"color: {hex_color};") # Style tab bar with scope accent color - if hasattr(self, 'tab_bar') and self.tab_bar: + if self.tab_bar: self.tab_bar.setStyleSheet(f""" QTabBar::tab {{ background-color: {self.color_scheme.to_hex(self.color_scheme.input_bg)}; color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; - padding: 8px 16px; + padding: 0px 16px; margin-right: 2px; border: none; border-radius: 4px 4px 0 0; + height: {CURRENT_LAYOUT.button_height}px; }} QTabBar::tab:selected {{ background-color: {hex_color}; color: white; }} QTabBar::tab:hover:!selected {{ - background-color: {hex_lighter}; + background-color: {accent_color.lighter(115).name()}; color: white; }} """) # Style step_editor elements - if hasattr(self, 'step_editor') and self.step_editor: + if self.step_editor: # Tree selection tree_style = self.get_scope_tree_selection_stylesheet() - if tree_style and hasattr(self.step_editor, 'hierarchy_tree') and self.step_editor.hierarchy_tree: + if tree_style and self.step_editor.hierarchy_tree: current_style = self.step_editor.hierarchy_tree.styleSheet() or "" - self.step_editor.hierarchy_tree.setStyleSheet(f"{current_style}\n{tree_style}") + self.step_editor.hierarchy_tree.setStyleSheet( + f"{current_style}\n{tree_style}" + ) + + # "Step Parameters" header label (may be None if render_header=False) + if self.step_editor.header_label is not None: + self.step_editor.header_label.setStyleSheet( + f"color: {hex_color}; font-weight: bold; font-size: 14px;" + ) - # "Step Parameters" header label - if hasattr(self.step_editor, 'header_label') and self.step_editor.header_label: - self.step_editor.header_label.setStyleSheet(f"color: {hex_color}; font-weight: bold; font-size: 14px;") + if self._scope_color_scheme: + self.step_editor.apply_scope_color_scheme(self._scope_color_scheme) # Style func_editor elements (Function Pattern tab) - if hasattr(self, 'func_editor') and self.func_editor: - # "Functions" header label - if hasattr(self.func_editor, 'header_label') and self.func_editor.header_label: - self.func_editor.header_label.setStyleSheet(f"color: {hex_color}; font-weight: bold; font-size: 14px;") + if self.func_editor: + # "Functions" header label (may be None if render_header=False) + if self.func_editor.header_label is not None: + self.func_editor.header_label.setStyleSheet( + f"color: {hex_color}; font-weight: bold; font-size: 14px;" + ) + + # Apply scope color scheme to function panes (for enableable styling and colors) + # Use inheritance - _scope_color_scheme is set by ScopedBorderMixin._init_scope_border() + if self._scope_color_scheme: + self.func_editor.set_scope_color_scheme(self._scope_color_scheme) + + # Show window flash when dual editor opens + if self._flash_overlay is None: + from pyqt_reactive.animation import WindowFlashOverlay + + self._flash_overlay = WindowFlashOverlay(self) + self._flash_overlay_cleaned = False + + super()._apply_scope_accent_styling() def _build_step_scope_id(self) -> str: - return ScopeTokenService.build_scope_id(self.orchestrator.plate_path, self.editing_step) - + return ScopeTokenService.build_scope_id( + self.orchestrator.plate_path, self.editing_step + ) + def create_step_tab(self): """Create the step settings tab (using dedicated widget).""" # Create step parameter editor widget with proper nested context @@ -354,25 +588,45 @@ def create_step_tab(self): scope_id = self._build_step_scope_id() with config_context(self.orchestrator.pipeline_config): # Pipeline level - with config_context(self.editing_step): # Step level + with config_context(self.editing_step): # Step level self.step_editor = StepParameterEditorWidget( self.editing_step, service_adapter=None, color_scheme=self.color_scheme, pipeline_config=self.orchestrator.pipeline_config, - scope_id=scope_id + scope_id=scope_id, # Same hierarchical scope_id as step editor + step_index=self._step_index, # Align scope styling with pipeline order + scope_accent_color=self._scope_accent_color, + render_header=False, # Don't render header - buttons will be managed externally + button_style="compact", # Borderless compact style to match function editor ) - # Connect parameter changes - use form manager signal for immediate response - self.step_editor.form_manager.parameter_changed.connect(self.on_form_parameter_changed) + # NOTE: parameter_changed connection is now handled automatically by BaseManagedWindow._connect_change_detection() + # which is called at the end of __init__. This automatically calls detect_changes() when any parameter changes. + # We still need on_form_parameter_changed() for function editor sync, so connect it here. + self.step_editor.form_manager.parameter_changed.connect( + self.on_form_parameter_changed + ) # NOTE: context_changed subscription REMOVED - FunctionListEditorWidget now subscribes # directly to ObjectState.on_resolved_changed, which is the proper mechanism for # reacting to resolved value changes from ANY ancestor (PipelineConfig, GlobalPipelineConfig) - # Subscribe to dirty state changes for window title updates + # Subscribe to state changes for window title updates self._dirty_title_callback = self._update_window_title self.step_editor.state.on_state_changed(self._dirty_title_callback) + self.step_editor.state.on_state_changed(self._dirty_detect_callback) + + def _update_title_on_resolved_changed(_: set) -> None: + self._update_window_title() + self.detect_changes() + + self.step_editor.state.on_resolved_changed(_update_title_on_resolved_changed) + + # CRITICAL: Set initial title now that ObjectState is available + # This ensures title shows immediately instead of waiting for first state change + self._update_window_title() + self.detect_changes() self.tab_widget.addTab(self.step_editor, "Step Settings") @@ -385,14 +639,16 @@ def create_function_tab(self): # CRITICAL: Pass editing_step for context hierarchy (Function → Step → Pipeline → Global) # CRITICAL: Use same hierarchical scope_id as step editor to isolate this step editor + its function panes scope_id = self._build_step_scope_id() - step_name = getattr(self.editing_step, 'name', 'unknown_step') + step_name = getattr(self.editing_step, "name", "unknown_step") self.func_editor = FunctionListEditorWidget( initial_functions=initial_functions, - step_identifier=step_name, + context_identifier=step_name, service_adapter=None, - step_instance=self.editing_step, # Pass step for lazy resolution context - scope_id=scope_id # Same hierarchical scope_id as step editor + scope_id=scope_id, # Same hierarchical scope_id as step editor + render_header=False, + button_style="compact", # Borderless compact style for tab row + scope_index=self._step_index, # Align scope styling with pipeline order ) # Store main window reference for orchestrator access (find it through parent chain) @@ -403,8 +659,14 @@ def create_function_tab(self): # SINGLE SOURCE OF TRUTH: Initialize function editor state from step self._sync_function_editor_from_step() + # Restore last selected dict-pattern key (persisted in ObjectState.metadata) + self.func_editor.apply_selected_pattern_key_from_state() + # Connect function pattern changes - self.func_editor.function_pattern_changed.connect(self._on_function_pattern_changed) + # Use DirectConnection to keep execution synchronous within atomic context + self.func_editor.function_pattern_changed.connect( + self._on_function_pattern_changed, type=Qt.ConnectionType.DirectConnection + ) self.tab_widget.addTab(self.func_editor, "Function Pattern") @@ -412,25 +674,52 @@ def _on_function_pattern_changed(self): """Handle function pattern changes from function editor.""" # Update step func from function editor - use current_pattern to get full pattern data current_pattern = self.func_editor.current_pattern + logger.debug( + "[FUNC_PATTERN] current_pattern type=%s value=%r", + type(current_pattern).__name__, + current_pattern, + ) # CRITICAL FIX: Use fresh imports to avoid unpicklable registry wrappers - if callable(current_pattern) and hasattr(current_pattern, '__module__'): + if callable(current_pattern): try: import importlib + module = importlib.import_module(current_pattern.__module__) current_pattern = getattr(module, current_pattern.__name__) except Exception: pass # Use original if refresh fails - self.editing_step.func = current_pattern + # ATOMIC: Coalesce function parameter change + step func update into single undo + # Without this, editing a function parameter creates two undo entries: + # one for the function parameter and one for the step's func pattern + state = self.step_editor.state if self.step_editor else None + state_func = ( + state.parameters.get("func") + if state is not None and "func" in state.parameters + else None + ) + step_func = getattr(self.editing_step, "func", None) + if state_func == current_pattern and step_func == current_pattern: + logger.debug("[FUNC_PATTERN] Ignoring no-op function pattern update") + return + + with ObjectStateRegistry.atomic("edit func"): + if step_func != current_pattern: + self.editing_step.func = current_pattern - # CRITICAL: Also update ObjectState so list item preview updates in real-time - # The step_editor's state tracks 'func' parameter - update it with the new pattern - if hasattr(self, 'step_editor') and self.step_editor and hasattr(self.step_editor, 'state'): - state = self.step_editor.state - if state and 'func' in state.parameters: - state.update_parameter('func', current_pattern) - logger.debug(f"Updated ObjectState 'func' parameter for real-time preview") + # CRITICAL: Also update ObjectState so list item preview updates in real-time + # The step_editor's state tracks 'func' parameter - update it with the new pattern + if state is not None: + if "func" in state.parameters and state_func != current_pattern: + state.update_parameter("func", current_pattern) + logger.debug( + f"Updated ObjectState 'func' parameter for real-time preview" + ) + logger.debug( + "[FUNC_PATTERN] ObjectState dirty_fields after update: %s", + state.dirty_fields, + ) self.detect_changes() logger.debug(f"Function pattern changed: {current_pattern}") @@ -445,7 +734,7 @@ def _get_event_bus(self): # Navigate up to find main window with service adapter current = self.parent() while current: - if hasattr(current, 'service_adapter'): + if current.service_adapter: return current.service_adapter.get_event_bus() current = current.parent() @@ -473,12 +762,14 @@ def _on_pipeline_changed(self, new_pipeline_steps: list): # Find our step in the new pipeline by matching scope_id # CRITICAL: Use scope_id matching (more robust than object identity) # The window's scope_id is "plate_path::functionstep_N", extract the token - window_scope_id = getattr(self, 'scope_id', None) + window_scope_id = self.scope_id if not window_scope_id: return # Extract step token from scope_id (e.g., "plate_path::functionstep_3" -> "functionstep_3") - window_step_token = window_scope_id.split('::')[-1] if '::' in window_scope_id else None + window_step_token = ( + window_scope_id.split("::")[-1] if "::" in window_scope_id else None + ) if not window_step_token: return @@ -486,7 +777,7 @@ def _on_pipeline_changed(self, new_pipeline_steps: list): updated_step = None new_index = None for i, step in enumerate(new_pipeline_steps): - step_token = getattr(step, '_scope_token', None) + step_token = getattr(step, "_scope_token", None) if step_token == window_step_token: updated_step = step new_index = i @@ -495,11 +786,15 @@ def _on_pipeline_changed(self, new_pipeline_steps: list): # Check if step position changed - refresh scope border styling only # (no need to repopulate widgets, just update colors) if new_index is not None and new_index != self._step_index: - logger.debug(f"Step moved from index {self._step_index} to {new_index} - refreshing scope border") + logger.debug( + f"Step moved from index {self._step_index} to {new_index} - refreshing scope border" + ) self._step_index = new_index + if self.step_editor: + self.step_editor.step_index = new_index + if self.func_editor: + self.func_editor.set_scope_index(new_index) self._refresh_scope_border() - # Re-apply accent styling to all children with updated color scheme - self._apply_accent_to_help_buttons() # Only refresh data if the step OBJECT was replaced in the pipeline # (e.g., from code editor saving a new step instance). @@ -507,13 +802,15 @@ def _on_pipeline_changed(self, new_pipeline_steps: list): # NOTE: We never replace editing_step with the pipeline step - editing_step # is a clone that preserves isolation for unsaved changes. if updated_step and updated_step is not self.original_step_reference: - logger.debug(f"Pipeline step object was replaced - syncing data for: {updated_step.name}") + logger.debug( + f"Pipeline step object was replaced - syncing data for: {updated_step.name}" + ) # Update reference to the new pipeline step self.original_step_reference = updated_step # Update function list editor with new func (this recreates panes) - if hasattr(self, 'func_editor') and self.func_editor and hasattr(updated_step, 'func'): + if self.func_editor and updated_step.func: self.func_editor._initialize_pattern_data(updated_step.func) self.func_editor._populate_function_list() @@ -568,9 +865,6 @@ def setup_connections(self): # Tab change tracking self.tab_widget.currentChanged.connect(self.on_tab_changed) - # Change detection - self.changes_detected.connect(self.on_changes_detected) - # CRITICAL: Connect to global event bus for cross-window updates # This is the OpenHCS "set and forget" pattern - one connection handles ALL sources event_bus = self._get_event_bus() @@ -582,7 +876,7 @@ def setup_connections(self): def _convert_step_func_to_list(self): """Convert step func to initial pattern format for function list editor.""" - if not hasattr(self.editing_step, 'func') or not self.editing_step.func: + if not self.editing_step.func: return [] # Return the step func directly - the function list editor will handle the conversion @@ -598,7 +892,7 @@ def _schedule_function_editor_sync(self): def _flush_function_editor_sync(self): """Run any pending function editor sync.""" - if not getattr(self, '_pending_function_editor_sync', False): + if not self._pending_function_editor_sync: return self._pending_function_editor_sync = False self._sync_function_editor_from_step() @@ -621,19 +915,14 @@ def _sync_function_editor_from_step(self): logger.debug("🔄 _sync_function_editor_from_step called") # Guard: Only sync if function editor exists - if not hasattr(self, 'func_editor') or self.func_editor is None: + if self.func_editor is None: logger.debug("⏭️ Function editor doesn't exist yet, skipping sync") return - # CRITICAL: The function editor already has access to step_instance and can resolve - # group_by using the same lazy resolution pattern as the function panes. - # Just trigger a refresh - the function editor will read from step_instance.processing_config.group_by - # with proper context resolution (including live context from other open windows). - self.func_editor.refresh_from_step_context() - - logger.debug(f"✅ Triggered function editor refresh from step context") - + # Trigger a refresh from the editor's authoritative ObjectState context. + self.func_editor.refresh_from_context() + logger.debug("✅ Triggered function editor refresh from context") def _find_main_window(self): """Find the main window through the parent chain.""" @@ -641,8 +930,8 @@ def _find_main_window(self): # Navigate up the parent chain to find OpenHCSMainWindow current = self.parent() while current: - # Check if this is the main window (has floating_windows attribute) - if hasattr(current, 'floating_windows') and hasattr(current, 'service_adapter'): + # Check if this is a main window (has floating_windows attribute) + if current.floating_windows and current.service_adapter: logger.debug(f"Found main window: {type(current).__name__}") return current current = current.parent() @@ -661,19 +950,35 @@ def _get_current_plate_from_pipeline_editor(self): current = self.parent() while current: # Check if this is a pipeline editor widget - if hasattr(current, 'current_plate') and hasattr(current, 'pipeline_steps'): - current_plate = getattr(current, 'current_plate', None) + try: + current.current_plate + current.pipeline_steps + current_plate = current.current_plate if current_plate: - logger.debug(f"Found current plate from pipeline editor: {current_plate}") + logger.debug( + f"Found current plate from pipeline editor: {current_plate}" + ) return current_plate + except AttributeError: + logger.debug( + f"Widget doesn't have current_plate/pipeline_steps attributes" + ) # Check children for pipeline editor widget for child in current.findChildren(QWidget): - if hasattr(child, 'current_plate') and hasattr(child, 'pipeline_steps'): - current_plate = getattr(child, 'current_plate', None) + try: + child.current_plate + child.pipeline_steps + current_plate = child.current_plate if current_plate: - logger.debug(f"Found current plate from pipeline editor child: {current_plate}") + logger.debug( + f"Found current plate from pipeline editor child: {current_plate}" + ) return current_plate + except AttributeError: + logger.debug( + f"Child widget doesn't have current_plate/pipeline_steps attributes" + ) current = current.parent() @@ -685,34 +990,34 @@ def _get_current_plate_from_pipeline_editor(self): return None # Old function pane methods removed - now using dedicated FunctionListEditorWidget - + def get_function_info(self) -> str: """ Get function information for display. - + Returns: Function information string """ - if not self.editing_step or not hasattr(self.editing_step, 'func'): + if not self.editing_step.func: return "No function assigned" - + func = self.editing_step.func - func_name = getattr(func, '__name__', 'Unknown Function') - func_module = getattr(func, '__module__', 'Unknown Module') - + func_name = getattr(func, "__name__", "Unknown Function") + func_module = getattr(func, "__module__", "Unknown Module") + info = f"Function: {func_name}\n" info += f"Module: {func_module}\n" - + # Add parameter info if available - if hasattr(self.editing_step, 'parameters'): + if self.editing_step.parameters: params = self.editing_step.parameters if params: info += f"\nParameters ({len(params)}):\n" for param_name, param_value in params.items(): info += f" {param_name}: {param_value}\n" - + return info - + def on_orchestrator_config_changed(self, plate_path: str, effective_config): """Handle orchestrator configuration changes for placeholder refresh. @@ -726,25 +1031,30 @@ def on_orchestrator_config_changed(self, plate_path: str, effective_config): """ # Only refresh if this is for our orchestrator if self.orchestrator and str(self.orchestrator.plate_path) == plate_path: - logger.debug(f"Step editor received orchestrator config change for {plate_path}") + logger.debug( + f"Step editor received orchestrator config change for {plate_path}" + ) # Update our stored pipeline_config reference to the orchestrator's current config self.pipeline_config = self.orchestrator.pipeline_config # Update the step editor's pipeline_config reference - if hasattr(self, 'step_editor') and self.step_editor: - self.step_editor.pipeline_config = self.orchestrator.pipeline_config - - # Update the form manager's context_obj to use the new pipeline config - if hasattr(self.step_editor, 'form_manager') and self.step_editor.form_manager: - # CRITICAL: Update context_obj for root form manager AND all nested managers - # Nested managers (e.g., processing_config) also have context_obj references that need updating - self._update_context_obj_recursively(self.step_editor.form_manager, self.orchestrator.pipeline_config) + self.step_editor.pipeline_config = self.orchestrator.pipeline_config + + # Update the form manager's context_obj to use the new pipeline config + if self.step_editor.form_manager: + # CRITICAL: Update context_obj for root form manager AND all nested managers + # Nested managers (e.g., processing_config) also have context_obj references that need updating + self._update_context_obj_recursively( + self.step_editor.form_manager, self.orchestrator.pipeline_config + ) - # Refresh placeholders to show new inherited values - # Use the same pattern as on_config_changed (line 466) - ObjectStateRegistry.increment_token() - logger.debug("Triggered global cross-window refresh after pipeline config change") + # Refresh placeholders to show new inherited values + # Use the same pattern as on_config_changed (line 466) + ObjectStateRegistry.increment_token() + logger.debug( + "Triggered global cross-window refresh after pipeline config change" + ) def _update_context_obj_recursively(self, form_manager, new_context_obj): """Recursively update context_obj for a form manager and all its nested managers. @@ -763,9 +1073,8 @@ def _update_context_obj_recursively(self, form_manager, new_context_obj): form_manager.context_obj = new_context_obj # Recursively update all nested managers - if hasattr(form_manager, 'nested_managers'): - for nested_name, nested_manager in form_manager.nested_managers.items(): - self._update_context_obj_recursively(nested_manager, new_context_obj) + for nested_name, nested_manager in form_manager.nested_managers.items(): + self._update_context_obj_recursively(nested_manager, new_context_obj) def on_form_parameter_changed(self, param_name: str, value): """Handle form parameter changes directly from form manager. @@ -784,13 +1093,15 @@ def on_form_parameter_changed(self, param_name: str, value): # Handle reset_all completion signal if param_name == "__reset_all_complete__": - logger.debug("🔄 Received reset_all_complete signal, syncing function editor") + logger.debug( + "🔄 Received reset_all_complete signal, syncing function editor" + ) self._schedule_function_editor_sync() return # param_name is now a full path like "processing_config.group_by" or just "name" # Parse the path to determine if it's a nested field - path_parts = param_name.split('.') + path_parts = param_name.split(".") logger.debug(f" path_parts={path_parts}") # Skip the first part if it's the form manager's field_id (type name like "FunctionStep") @@ -805,11 +1116,12 @@ def on_form_parameter_changed(self, param_name: str, value): field_name = path_parts[0] # CRITICAL FIX: For function parameters, use fresh imports to avoid unpicklable registry wrappers - if field_name == 'func' and callable(value) and hasattr(value, '__module__'): + if field_name == "func" and callable(value): try: import importlib + module = importlib.import_module(value.__module__) - value = getattr(module, value.__name__) + value = module.__dict__.get(value.__name__) except Exception: pass # Use original if refresh fails @@ -817,10 +1129,17 @@ def on_form_parameter_changed(self, param_name: str, value): # don't replace the entire lazy dataclass - instead update individual fields # This preserves lazy resolution for fields that weren't changed if is_dataclass(value) and not isinstance(value, type): - logger.debug(f"📦 {field_name} is a nested dataclass, updating fields individually") - existing_config = getattr(self.editing_step, field_name, None) - if existing_config is not None and hasattr(existing_config, '_resolve_field_value'): + logger.debug( + f"📦 {field_name} is a nested dataclass, updating fields individually" + ) + existing_config = self.editing_step + try: + existing_config._resolve_field_value logger.debug(f"✅ {field_name} is lazy, preserving lazy resolution") + except AttributeError: + logger.debug( + f"Config doesn't have _resolve_field_value, updating directly" + ) for field in fields(value): raw_value = object.__getattribute__(value, field.name) object.__setattr__(existing_config, field.name, raw_value) @@ -839,23 +1158,28 @@ def on_form_parameter_changed(self, param_name: str, value): # SINGLE SOURCE OF TRUTH: Always sync function editor from step (batched) logger.debug(f" 🔄 Scheduling function editor sync after {param_name} change") self._schedule_function_editor_sync() - + def on_tab_changed(self, index: int): """Handle tab changes.""" tab_names = ["step", "function"] if 0 <= index < len(tab_names): self.current_tab = tab_names[index] logger.debug(f"Tab changed to: {self.current_tab}") - + def detect_changes(self): - """Detect if changes have been made.""" - current_snapshot = self._serialize_current_form_state() - baseline_snapshot = getattr(self, '_baseline_snapshot', None) - has_changes = current_snapshot != baseline_snapshot + """Detect if changes have been made using ObjectState's dirty tracking. + + This replaces the old snapshot-based approach with ObjectState's built-in + dirty tracking, which automatically detects changes to any parameter + (including nested fields) by comparing current values to saved baseline. + """ + # Use ObjectState's dirty tracking instead of custom snapshot comparison + has_changes = bool(self.state.is_raw_dirty) if self.state else False logger.debug(f"🔍 DETECT_CHANGES:") - logger.debug(f" current_snapshot={current_snapshot}") - logger.debug(f" baseline_snapshot={baseline_snapshot}") + logger.debug( + f" dirty_fields={self.state.dirty_fields if self.state else 'no state'}" + ) logger.debug(f" has_changes={has_changes}") if has_changes != self.has_changes: @@ -864,16 +1188,14 @@ def detect_changes(self): self.changes_detected.emit(has_changes) else: logger.debug(f" ⏭️ No change in has_changes state") - + def on_changes_detected(self, has_changes: bool): """Handle changes detection.""" - if has_changes: - self.changes_label.setText("● Unsaved changes") - self.save_button.setEnabled(True) - else: - self.changes_label.setText("") - self.save_button.setEnabled(False) - + # Enable/disable save button based on changes + self.save_button.setEnabled(has_changes) + # Update save button text to show asterisk when there are unsaved changes + self._update_save_button_text() + def save_edit(self, *, close_window=True): """Save the edited step. If close_window is True, close after saving; else, keep open.""" try: @@ -882,14 +1204,15 @@ def save_edit(self, *, close_window=True): if self.func_editor: current_pattern = self.func_editor.current_pattern - # CRITICAL FIX: Use fresh imports to avoid unpicklable registry wrappers - if callable(current_pattern) and hasattr(current_pattern, '__module__'): - try: - import importlib - module = importlib.import_module(current_pattern.__module__) - current_pattern = getattr(module, current_pattern.__name__) - except Exception: - pass # Use original if refresh fails + # CRITICAL FIX: Use fresh imports to avoid unpicklable registry wrappers + if callable(current_pattern): + try: + import importlib + + module = importlib.import_module(current_pattern.__module__) + current_pattern = module.__dict__.get(current_pattern.__name__) + except Exception: + pass # Use original if refresh fails self.editing_step.func = current_pattern logger.debug(f"Synced function pattern before save: {current_pattern}") @@ -898,29 +1221,34 @@ def save_edit(self, *, close_window=True): # This ensures nested dataclass field values are properly saved to the step object for tab_index in range(self.tab_widget.count()): tab_widget = self.tab_widget.widget(tab_index) - if hasattr(tab_widget, 'state') and tab_widget.state: + if tab_widget and hasattr(tab_widget, "state") and tab_widget.state: # Get current values from this tab's state current_values = tab_widget.state.get_current_values() - # Apply values to the editing step + # Apply values to editing step for param_name, value in current_values.items(): - if hasattr(self.editing_step, param_name): - setattr(self.editing_step, param_name, value) - logger.debug(f"Applied {param_name}={value} to editing step") + setattr(self.editing_step, param_name, value) + logger.debug(f"Applied {param_name}={value} to editing step") # Validate step - step_name = getattr(self.editing_step, 'name', None) + step_name = self.editing_step.name if not step_name or not step_name.strip(): - QMessageBox.warning(self, "Validation Error", "Step name cannot be empty.") + QMessageBox.warning( + self, "Validation Error", "Step name cannot be empty." + ) return # CRITICAL FIX: For existing steps, apply changes to original step object # This ensures the pipeline gets the updated step with the same object identity - logger.info(f"💾 Save: is_new={self.is_new}, original_step_reference={self.original_step_reference is not None}") + logger.info( + f"💾 Save: is_new={self.is_new}, original_step_reference={self.original_step_reference is not None}" + ) if self.original_step_reference is not None: # Editing existing step - logger.info(f"💾 Editing existing step: {getattr(self.original_step_reference, 'name', 'Unknown')}") + logger.info( + f"💾 Editing existing step: {self.original_step_reference.name}" + ) self._apply_changes_to_original() step_to_save = self.original_step_reference else: @@ -934,30 +1262,70 @@ def save_edit(self, *, close_window=True): self._update_save_button_text() # Emit signals and call callback - logger.info(f"💾 Emitting step_saved signal for: {getattr(step_to_save, 'name', 'Unknown')}") + logger.info( + f"💾 Emitting step_saved signal for: {getattr(step_to_save, 'name', 'Unknown')}" + ) self.step_saved.emit(step_to_save) if self.on_save_callback: logger.info(f"💾 Calling on_save_callback") self.on_save_callback(step_to_save) - # After a successful save, refresh the baseline snapshot used for change detection. - # This ensures Shift+Save (which keeps the window open) clears the "Unsaved changes" label. + # After a successful save, update original_step and detect changes + # ObjectState.mark_saved() is called by accept() or _mark_saved_and_refresh_all() self.original_step = self._clone_step(self.editing_step) - self._baseline_snapshot = self._serialize_for_change_detection(self.editing_step) - self.detect_changes() # UNIFIED: Both paths share same logic, differ only in whether to close window if close_window: self.accept() # Marks saved + unregisters + cleans up + closes else: self._mark_saved_and_refresh_all() # Marks saved + refreshes, but stays open - logger.debug(f"Step saved: {getattr(step_to_save, 'name', 'Unknown')}") + + # Detect changes after marking saved (should show no changes now) + self.detect_changes() except Exception as e: logger.error(f"Failed to save step: {e}") QMessageBox.critical(self, "Save Error", f"Failed to save step:\n{e}") + def select_and_scroll_to_field(self, field_path: str) -> None: + logger.debug(f"[SCROLL] select_and_scroll_to_field called with: {field_path!r}") + if not field_path: + logger.debug("[SCROLL] field_path is falsy, returning early") + return + + from objectstate import ObjectStateRegistry + + if not self.scope_id: + return + + state = ObjectStateRegistry.get_by_scope(self.scope_id) + if not state or state.object_instance is None: + return + + is_step = isinstance(state.object_instance, FunctionStep) + + # If the navigation target is the function pattern, use the function tab. + # This avoids fighting tab selection done by time-travel navigation. + if is_function_field_path(field_path): + if self.func_editor is None: + return + if self.tab_widget: + self.tab_widget.setCurrentIndex(1) + self.func_editor.select_and_scroll_to_field(field_path) + return + + if is_step and self.step_editor: + if self.tab_widget: + self.tab_widget.setCurrentIndex(0) + self.step_editor.select_and_scroll_to_field(field_path) + return + + if self.func_editor: + if self.tab_widget: + self.tab_widget.setCurrentIndex(1) + self.func_editor.select_and_scroll_to_field(field_path) + def _apply_changes_to_original(self): """Apply all changes from editing_step to original_step_reference.""" if self.original_step_reference is None: @@ -974,102 +1342,32 @@ def _apply_changes_to_original(self): # This ensures optional dataclass attributes like step_materialization_config are copied for attr_name in dir(self.editing_step): # Skip private/magic attributes and methods - if not attr_name.startswith('_') and not callable(getattr(self.editing_step, attr_name, None)): - if hasattr(self.editing_step, attr_name) and hasattr(self.original_step_reference, attr_name): + if not attr_name.startswith("_") and not callable( + getattr(self.editing_step, attr_name, None) + ): + try: value = getattr(self.editing_step, attr_name) setattr(self.original_step_reference, attr_name, value) logger.debug(f"Copied attribute {attr_name}: {value}") + except AttributeError: + pass logger.debug("Applied changes to original step object") def _clone_step(self, step): """Clone a step object using deep copy.""" import copy - return copy.deepcopy(step) - - _CHANGE_DETECTION_EXCLUDE_PARAMS = {'kwargs'} - - def _serialize_for_change_detection(self, step): - """Create a normalized snapshot of the step for change detection.""" - params = UnifiedParameterAnalyzer.analyze(step) - snapshot = {} - for name, param_info in params.items(): - if name in self._CHANGE_DETECTION_EXCLUDE_PARAMS: - continue - current_value = getattr(step, name, param_info.default_value) - snapshot[name] = self._normalize_value_for_change_detection(current_value) - return snapshot - - def _serialize_current_form_state(self): - """Serialize a snapshot of the step using live form values (including nested dataclasses).""" - import copy - - temp_step = copy.deepcopy(self.editing_step) - - # NOTE: We DON'T override func from func_editor.current_pattern here because: - # 1. current_pattern returns the pattern data structure (list of tuples), not the func value - # 2. editing_step.func should already be kept in sync by the function editor's signals - # 3. Using current_pattern would cause serialization mismatch with baseline - - for tab_index in range(self.tab_widget.count()): - tab_widget = self.tab_widget.widget(tab_index) - state = getattr(tab_widget, 'state', None) - if not state: - continue - - current_values = state.get_current_values() - for param_name, value in current_values.items(): - if hasattr(temp_step, param_name): - setattr(temp_step, param_name, value) - - return self._serialize_for_change_detection(temp_step) - - def _normalize_value_for_change_detection(self, value): - """Normalize complex values into comparison-friendly primitives.""" - if value is None: - return None - - if isinstance(value, (str, int, float, bool)): - return value - if isinstance(value, Path): - return str(value) - - if isinstance(value, list): - return [self._normalize_value_for_change_detection(v) for v in value] - - if isinstance(value, tuple): - return tuple(self._normalize_value_for_change_detection(v) for v in value) - - if isinstance(value, dict): - return {k: self._normalize_value_for_change_detection(v) for k, v in value.items()} - - if hasattr(value, 'value') and hasattr(value, 'name'): - return value.value - - if callable(value): - module = getattr(value, '__module__', '') - qualname = getattr(value, '__qualname__', getattr(value, '__name__', repr(value))) - return f"{module}.{qualname}" - - if is_dataclass(value): - is_lazy_dataclass = get_base_type_for_lazy(type(value)) is not None - normalized = {} - for field in fields(value): - if is_lazy_dataclass: - raw_field_value = object.__getattribute__(value, field.name) - else: - raw_field_value = getattr(value, field.name) - normalized[field.name] = self._normalize_value_for_change_detection(raw_field_value) - return normalized + return copy.deepcopy(step) - return repr(value) + # NOTE: Snapshot-based change detection removed - now using ObjectState.dirty_fields + # This is simpler, more reliable, and automatically handles nested fields def _create_new_step(self): """Create a new empty step.""" return FunctionStep( func=[], # Start with empty function list - name="New_Step" + name="New_Step", ) def cancel_edit(self): @@ -1078,23 +1376,19 @@ def cancel_edit(self): self.reject() def reject(self): - """Handle dialog rejection (Cancel button or Escape key).""" + """Handle dialog rejection (Cancel button or Escape key). + + Restores ObjectState to last saved state, undoing all unsaved changes. + """ # No confirmation needed - time travel allows recovery of any state self.step_cancelled.emit() - # CRITICAL: Before restore_saved() is called by super(), get the saved func value - # so we can reset the editing_step and func_editor to match - if hasattr(self, 'step_editor') and self.step_editor and hasattr(self.step_editor, 'state'): - state = self.step_editor.state - if state and 'func' in state._saved_parameters: - saved_func = state._saved_parameters['func'] - # Restore editing_step.func to saved value (undo any live changes) - self.editing_step.func = saved_func - logger.debug(f"Restored editing_step.func to saved value") - logger.info("🔍 DualEditorWindow: About to call super().reject()") - super().reject() # BaseFormDialog handles unregistration + + # CRITICAL: super().reject() calls state.restore_saved() to undo ALL unsaved changes + # This restores all parameters (including func) to last saved state automatically + super().reject() # BaseFormDialog handles state restoration + unregistration # CRITICAL: Trigger global refresh AFTER unregistration so other windows # re-collect live context without this cancelled window's values @@ -1110,6 +1404,7 @@ def closeEvent(self, event): if self.step_editor is not None: self.step_editor.tree_helper.cleanup_subscriptions() self.step_editor.state.off_state_changed(self._dirty_title_callback) + self.step_editor.state.off_state_changed(self._dirty_detect_callback) super().closeEvent(event) # BaseFormDialog handles unregistration diff --git a/openhcs/pyqt_gui/windows/help_windows.py b/openhcs/pyqt_gui/windows/help_windows.py index cdf9c6c67..a0006b987 100644 --- a/openhcs/pyqt_gui/windows/help_windows.py +++ b/openhcs/pyqt_gui/windows/help_windows.py @@ -3,8 +3,15 @@ import logging from typing import Union, Callable, Optional from PyQt6.QtWidgets import ( - QDialog, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, - QTextEdit, QScrollArea, QWidget, QMessageBox + QDialog, + QVBoxLayout, + QHBoxLayout, + QPushButton, + QLabel, + QTextEdit, + QScrollArea, + QWidget, + QMessageBox, ) from PyQt6.QtCore import Qt @@ -18,8 +25,13 @@ class BaseHelpWindow(QDialog): """Base class for all PyQt6 help windows - reuses Textual TUI help logic.""" - - def __init__(self, title: str = "Help", color_scheme: Optional[ColorScheme] = None, parent=None): + + def __init__( + self, + title: str = "Help", + color_scheme: Optional[ColorScheme] = None, + parent=None, + ): super().__init__(parent) # Initialize color scheme and style generator @@ -34,34 +46,43 @@ def __init__(self, title: str = "Help", color_scheme: Optional[ColorScheme] = No # Apply centralized styling self.setStyleSheet(self.style_generator.generate_dialog_style()) - + def setup_ui(self): """Setup the base help window UI.""" layout = QVBoxLayout(self) - + # Content area (to be filled by subclasses) self.content_area = QScrollArea() self.content_area.setWidgetResizable(True) - self.content_area.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) - self.content_area.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) + self.content_area.setVerticalScrollBarPolicy( + Qt.ScrollBarPolicy.ScrollBarAsNeeded + ) + self.content_area.setHorizontalScrollBarPolicy( + Qt.ScrollBarPolicy.ScrollBarAsNeeded + ) layout.addWidget(self.content_area) - + # Close button button_layout = QHBoxLayout() button_layout.addStretch() - + close_btn = QPushButton("Close") close_btn.clicked.connect(self.close) button_layout.addWidget(close_btn) - + layout.addLayout(button_layout) class DocstringHelpWindow(BaseHelpWindow): """Help window for functions and classes - reuses Textual TUI DocstringExtractor.""" - - def __init__(self, target: Union[Callable, type], title: Optional[str] = None, - color_scheme: Optional[ColorScheme] = None, parent=None): + + def __init__( + self, + target: Union[Callable, type], + title: Optional[str] = None, + color_scheme: Optional[ColorScheme] = None, + parent=None, + ): self.target = target # REUSE Textual TUI docstring extraction logic @@ -69,14 +90,14 @@ def __init__(self, target: Union[Callable, type], title: Optional[str] = None, # Generate title from target if not provided if title is None: - if hasattr(target, '__name__'): + if hasattr(target, "__name__"): title = f"Help: {target.__name__}" else: title = "Help" super().__init__(title, color_scheme, parent) self.populate_content() - + def populate_content(self): """Populate the help content with minimal styling.""" content_widget = QWidget() @@ -88,8 +109,12 @@ def populate_content(self): if self.docstring_info.summary: summary_label = QLabel(self.docstring_info.summary) summary_label.setWordWrap(True) - summary_label.setTextInteractionFlags(Qt.TextInteractionFlag.NoTextInteraction) - summary_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px;") + summary_label.setTextInteractionFlags( + Qt.TextInteractionFlag.NoTextInteraction + ) + summary_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px;" + ) layout.addWidget(summary_label) # Full description @@ -97,56 +122,84 @@ def populate_content(self): desc_label = QLabel(self.docstring_info.description) desc_label.setWordWrap(True) desc_label.setTextInteractionFlags(Qt.TextInteractionFlag.NoTextInteraction) - desc_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px;") + desc_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px;" + ) layout.addWidget(desc_label) - + # Parameters section if self.docstring_info.parameters: params_label = QLabel("Parameters:") - params_label.setTextInteractionFlags(Qt.TextInteractionFlag.NoTextInteraction) - params_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; font-size: 14px; font-weight: bold; margin-top: 8px;") + params_label.setTextInteractionFlags( + Qt.TextInteractionFlag.NoTextInteraction + ) + params_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; font-size: 14px; font-weight: bold; margin-top: 8px;" + ) layout.addWidget(params_label) for param_name, param_desc in self.docstring_info.parameters.items(): # Parameter name name_label = QLabel(f"• {param_name}") - name_label.setTextInteractionFlags(Qt.TextInteractionFlag.NoTextInteraction) - name_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px; margin-left: 5px; margin-top: 3px;") + name_label.setTextInteractionFlags( + Qt.TextInteractionFlag.NoTextInteraction + ) + name_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px; margin-left: 5px; margin-top: 3px;" + ) layout.addWidget(name_label) # Parameter description if param_desc: desc_label = QLabel(param_desc) desc_label.setWordWrap(True) - desc_label.setTextInteractionFlags(Qt.TextInteractionFlag.NoTextInteraction) - desc_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px; margin-left: 20px;") + desc_label.setTextInteractionFlags( + Qt.TextInteractionFlag.NoTextInteraction + ) + desc_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px; margin-left: 20px;" + ) layout.addWidget(desc_label) - + # Returns section if self.docstring_info.returns: returns_label = QLabel("Returns:") - returns_label.setTextInteractionFlags(Qt.TextInteractionFlag.NoTextInteraction) - returns_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; font-size: 14px; font-weight: bold; margin-top: 8px;") + returns_label.setTextInteractionFlags( + Qt.TextInteractionFlag.NoTextInteraction + ) + returns_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; font-size: 14px; font-weight: bold; margin-top: 8px;" + ) layout.addWidget(returns_label) returns_desc = QLabel(self.docstring_info.returns) returns_desc.setWordWrap(True) - returns_desc.setTextInteractionFlags(Qt.TextInteractionFlag.NoTextInteraction) - returns_desc.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px; margin-left: 5px;") + returns_desc.setTextInteractionFlags( + Qt.TextInteractionFlag.NoTextInteraction + ) + returns_desc.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; font-size: 12px; margin-left: 5px;" + ) layout.addWidget(returns_desc) - + # Examples section if self.docstring_info.examples: examples_label = QLabel("Examples:") - examples_label.setTextInteractionFlags(Qt.TextInteractionFlag.NoTextInteraction) - examples_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; font-size: 14px; font-weight: bold; margin-top: 8px;") + examples_label.setTextInteractionFlags( + Qt.TextInteractionFlag.NoTextInteraction + ) + examples_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; font-size: 14px; font-weight: bold; margin-top: 8px;" + ) layout.addWidget(examples_label) examples_text = QTextEdit() examples_text.setPlainText(self.docstring_info.examples) examples_text.setReadOnly(True) examples_text.setMaximumHeight(150) - examples_text.setTextInteractionFlags(Qt.TextInteractionFlag.NoTextInteraction) + examples_text.setTextInteractionFlags( + Qt.TextInteractionFlag.NoTextInteraction + ) examples_text.setStyleSheet(f""" QTextEdit {{ background-color: transparent; @@ -160,14 +213,13 @@ def populate_content(self): }} """) layout.addWidget(examples_text) - + layout.addStretch() self.content_area.setWidget(content_widget) # Auto-size to content self.adjustSize() # Set reasonable min/max sizes - self.setMinimumSize(400, 200) self.setMaximumSize(800, 600) @@ -178,16 +230,22 @@ class HelpWindowManager: _help_window = None @classmethod - def show_docstring_help(cls, target: Union[Callable, type], title: Optional[str] = None, parent=None): + def show_docstring_help( + cls, target: Union[Callable, type], title: Optional[str] = None, parent=None + ): """Show help for a function or class - reuses Textual TUI extraction logic.""" try: # Check if existing window is still valid - if cls._help_window and hasattr(cls._help_window, 'isVisible'): + if cls._help_window and hasattr(cls._help_window, "isVisible"): try: if not cls._help_window.isHidden(): cls._help_window.target = target - cls._help_window.docstring_info = DocstringExtractor.extract(target) - cls._help_window.setWindowTitle(title or f"Help: {getattr(target, '__name__', 'Unknown')}") + cls._help_window.docstring_info = DocstringExtractor.extract( + target + ) + cls._help_window.setWindowTitle( + title or f"Help: {getattr(target, '__name__', 'Unknown')}" + ) cls._help_window.populate_content() cls._help_window.raise_() cls._help_window.activateWindow() @@ -205,7 +263,13 @@ def show_docstring_help(cls, target: Union[Callable, type], title: Optional[str] QMessageBox.warning(parent, "Help Error", f"Failed to show help: {e}") @classmethod - def show_parameter_help(cls, param_name: str, param_description: str, param_type: type = None, parent=None): + def show_parameter_help( + cls, + param_name: str, + param_description: str, + param_type: type = None, + parent=None, + ): """Show help for a parameter - creates a fake docstring object and uses DocstringHelpWindow.""" try: # Create a fake docstring info object for the parameter @@ -220,17 +284,21 @@ class FakeDocstringInfo: examples: str = "" # Build parameter display - type_str = f" ({getattr(param_type, '__name__', str(param_type))})" if param_type else "" + type_str = ( + f" ({getattr(param_type, '__name__', str(param_type))})" + if param_type + else "" + ) fake_info = FakeDocstringInfo( summary=f"• {param_name}{type_str}", description=param_description or "No description available", parameters={}, returns="", - examples="" + examples="", ) # Check if existing window is still valid - if cls._help_window and hasattr(cls._help_window, 'isVisible'): + if cls._help_window and hasattr(cls._help_window, "isVisible"): try: if not cls._help_window.isHidden(): cls._help_window.docstring_info = fake_info @@ -247,7 +315,9 @@ class FakeDocstringInfo: class FakeTarget: __name__ = param_name - cls._help_window = DocstringHelpWindow(FakeTarget, title=f"Parameter: {param_name}", parent=parent) + cls._help_window = DocstringHelpWindow( + FakeTarget, title=f"Parameter: {param_name}", parent=parent + ) cls._help_window.docstring_info = fake_info cls._help_window.populate_content() cls._help_window.show() @@ -259,11 +329,15 @@ class FakeTarget: class HelpableWidget: """Mixin class to add help functionality to PyQt6 widgets - mirrors Textual TUI.""" - + def show_function_help(self, target: Union[Callable, type]) -> None: """Show help window for a function or class.""" HelpWindowManager.show_docstring_help(target, parent=self) - - def show_parameter_help(self, param_name: str, param_description: str, param_type: type = None) -> None: + + def show_parameter_help( + self, param_name: str, param_description: str, param_type: type = None + ) -> None: """Show help window for a parameter.""" - HelpWindowManager.show_parameter_help(param_name, param_description, param_type, parent=self) + HelpWindowManager.show_parameter_help( + param_name, param_description, param_type, parent=self + ) diff --git a/openhcs/pyqt_gui/windows/managed_windows.py b/openhcs/pyqt_gui/windows/managed_windows.py new file mode 100644 index 000000000..104d8e87a --- /dev/null +++ b/openhcs/pyqt_gui/windows/managed_windows.py @@ -0,0 +1,190 @@ +""" +Managed window implementations using WindowManager.show_or_focus(). + +Each window is created by a factory function passed to WindowManager. +""" + +from PyQt6.QtWidgets import QDialog, QVBoxLayout + + +class PlateManagerWindow(QDialog): + def __init__(self, main_window, service_adapter): + super().__init__(main_window) + self.main_window = main_window + self.service_adapter = service_adapter + self.setWindowTitle("Plate Manager") + self.setModal(False) + self.resize(600, 400) + + from openhcs.pyqt_gui.widgets.plate_manager import PlateManagerWidget + + layout = QVBoxLayout(self) + self.widget = PlateManagerWidget( + self.service_adapter, + self.service_adapter.get_current_color_scheme(), + ) + layout.addWidget(self.widget) + self._setup_connections() + + def _setup_connections(self): + self.widget.global_config_changed.connect( + lambda: self.main_window.on_config_changed( + self.service_adapter.get_global_config() + ) + ) + + if hasattr(self.main_window, "status_bar"): + self._setup_progress_signals() + + self._connect_to_pipeline_editor() + + def _setup_progress_signals(self): + from PyQt6.QtWidgets import QProgressBar + + if not hasattr(self.main_window, "_status_progress_bar"): + self.main_window._status_progress_bar = QProgressBar() + self.main_window._status_progress_bar.setMaximumWidth(200) + self.main_window._status_progress_bar.setVisible(False) + self.main_window.status_bar.addPermanentWidget( + self.main_window._status_progress_bar + ) + + self.widget.progress_started.connect( + self.main_window._on_plate_progress_started + ) + self.widget.progress_updated.connect( + self.main_window._on_plate_progress_updated + ) + self.widget.progress_finished.connect( + self.main_window._on_plate_progress_finished + ) + + def _connect_to_pipeline_editor(self): + from pyqt_reactive.services.window_manager import WindowManager + + pipeline_window = WindowManager._scoped_windows.get("pipeline_editor") + if pipeline_window: + from openhcs.pyqt_gui.widgets.pipeline_editor import PipelineEditorWidget + + pipeline_widget = pipeline_window.widget + self.widget.plate_selected.connect(pipeline_widget.set_current_plate) + self.widget.orchestrator_config_changed.connect( + pipeline_widget.on_orchestrator_config_changed + ) + self.widget.set_pipeline_editor(pipeline_widget) + pipeline_widget.plate_manager = self.widget + + +class PipelineEditorWindow(QDialog): + def __init__(self, main_window, service_adapter): + super().__init__(main_window) + self.main_window = main_window + self.service_adapter = service_adapter + self.setWindowTitle("Pipeline Editor") + self.setModal(False) + self.resize(800, 600) + + from openhcs.pyqt_gui.widgets.pipeline_editor import PipelineEditorWidget + + layout = QVBoxLayout(self) + self.widget = PipelineEditorWidget( + self.service_adapter, + self.service_adapter.get_current_color_scheme(), + ) + layout.addWidget(self.widget) + self._setup_connections() + + def _setup_connections(self): + from pyqt_reactive.services.window_manager import WindowManager + + plate_window = WindowManager._scoped_windows.get("plate_manager") + if plate_window: + plate_widget = plate_window.widget + plate_widget.plate_selected.connect(self.widget.set_current_plate) + plate_widget.orchestrator_config_changed.connect( + self.widget.on_orchestrator_config_changed + ) + self.widget.plate_manager = plate_widget + plate_widget.set_pipeline_editor(self.widget) + + +class ImageBrowserWindow(QDialog): + def __init__(self, main_window, service_adapter): + super().__init__(main_window) + self.main_window = main_window + self.service_adapter = service_adapter + self.setWindowTitle("Image Browser") + self.setModal(False) + self.resize(900, 600) + + from openhcs.pyqt_gui.widgets.image_browser import ImageBrowserWidget + + layout = QVBoxLayout(self) + self.widget = ImageBrowserWidget( + orchestrator=None, + color_scheme=self.service_adapter.get_current_color_scheme(), + ) + layout.addWidget(self.widget) + self._setup_connections() + + def _setup_connections(self): + from pyqt_reactive.services.window_manager import WindowManager + + plate_window = WindowManager._scoped_windows.get("plate_manager") + if plate_window and hasattr(plate_window.widget, "plate_selected"): + plate_widget = plate_window.widget + plate_widget.plate_selected.connect( + lambda: self._update_orchestrator(plate_widget) + ) + self._update_orchestrator(plate_widget) + + def _update_orchestrator(self, plate_widget): + orchestrator = plate_widget.get_selected_orchestrator() + if orchestrator: + self.widget.set_orchestrator(orchestrator) + + +class LogViewerWindowWrapper(QDialog): + def __init__(self, main_window, service_adapter): + super().__init__(main_window) + self.main_window = main_window + self.service_adapter = service_adapter + self.setWindowTitle("Log Viewer") + self.setModal(False) + self.resize(900, 700) + + from pyqt_reactive.widgets.log_viewer import LogViewerWindow + + layout = QVBoxLayout(self) + self.widget = LogViewerWindow( + self.main_window.file_manager, self.service_adapter + ) + layout.addWidget(self.widget) + + +class ZMQServerManagerWindow(QDialog): + def __init__(self, main_window, service_adapter): + super().__init__(main_window) + self.main_window = main_window + self.service_adapter = service_adapter + self.setWindowTitle("ZMQ Server Manager") + self.setModal(False) + self.resize(600, 400) + + from PyQt6.QtWidgets import QVBoxLayout + from openhcs.pyqt_gui.widgets.shared.zmq_server_manager import ( + ZMQServerManagerWidget, + ) + + from openhcs.core.config import get_all_streaming_ports + + layout = QVBoxLayout(self) + ports_to_scan = get_all_streaming_ports(num_ports_per_type=10) + + self.widget = ZMQServerManagerWidget( + ports_to_scan=ports_to_scan, + title="ZMQ Servers (Execution + Napari + Fiji)", + style_generator=self.service_adapter.get_style_generator(), + ) + layout.addWidget(self.widget) + self.widget.log_file_opened.connect(self.main_window._open_log_file_in_viewer) diff --git a/openhcs/pyqt_gui/windows/plate_viewer_window.py b/openhcs/pyqt_gui/windows/plate_viewer_window.py index 6a2e1b383..03409d569 100644 --- a/openhcs/pyqt_gui/windows/plate_viewer_window.py +++ b/openhcs/pyqt_gui/windows/plate_viewer_window.py @@ -8,15 +8,19 @@ from typing import Optional from PyQt6.QtWidgets import ( - QVBoxLayout, QHBoxLayout, QPushButton, - QTabWidget, QWidget, QLabel + QVBoxLayout, + QHBoxLayout, + QPushButton, + QTabWidget, + QWidget, + QLabel, ) from PyQt6.QtCore import Qt from PyQt6.QtGui import QFont from pyqt_reactive.theming import ColorScheme from pyqt_reactive.theming import StyleSheetGenerator -from openhcs.pyqt_gui.windows.base_form_dialog import BaseFormDialog +from pyqt_reactive.widgets.shared import BaseFormDialog logger = logging.getLogger(__name__) @@ -33,30 +37,54 @@ class PlateViewerWindow(BaseFormDialog): Only ONE PlateViewerWindow per plate can be open at a time. """ - def __init__(self, orchestrator, color_scheme: Optional[ColorScheme] = None, parent=None): + def __init__(self, orchestrator, parent=None): """ Initialize plate viewer window. Args: orchestrator: PipelineOrchestrator instance - color_scheme: Color scheme for styling parent: Parent widget """ super().__init__(parent) self.orchestrator = orchestrator - self.color_scheme = color_scheme or ColorScheme() - self.style_gen = StyleSheetGenerator(self.color_scheme) # scope_id for singleton behavior - one viewer per plate # Use ::plate_viewer suffix to avoid conflicts with ConfigWindow (which uses just plate_path) - self.scope_id = f"{orchestrator.plate_path}::plate_viewer" if orchestrator else None + self.scope_id = ( + f"{orchestrator.plate_path}::plate_viewer" if orchestrator else None + ) # Store plate path for styling (without suffix) so border matches plate's ConfigWindow self._style_scope_id = str(orchestrator.plate_path) if orchestrator else None + # CRITICAL: Initialize scope-based styling BEFORE creating child widgets + # This sets self._scope_accent_color for use in this class + if self._style_scope_id: + self._init_scope_border() + + # Get scope accent color and create color scheme from it + from pyqt_reactive.services.scope_color_service import ScopeColorService + + accent_color = ScopeColorService.instance().get_accent_color( + self._style_scope_id + ) + + # Create color scheme with accent color as text_accent (convert QColor to RGB tuple) + if accent_color: + self.color_scheme = ColorScheme() + self.color_scheme.text_accent = ( + accent_color.red(), + accent_color.green(), + accent_color.blue(), + ) + else: + self.color_scheme = ColorScheme() + + self.style_gen = StyleSheetGenerator(self.color_scheme) + plate_name = orchestrator.plate_path.name if orchestrator else "Unknown" self.setWindowTitle(f"Plate Viewer - {plate_name}") - self.setMinimumSize(1200, 800) - self.resize(1400, 900) + self.setMinimumSize(800, 600) + self.resize(1200, 800) # Make floating window with Dialog hint so tiling WMs don't fullscreen it # Qt.WindowType.Window alone strips the Dialog flag, causing tiling WMs to tile/fullscreen @@ -109,13 +137,20 @@ def _setup_ui(self): title_label = QLabel(title_text) title_label.setFont(QFont("Arial", 12, QFont.Weight.Bold)) - title_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};") + title_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};" + ) title_label.setWordWrap(False) # Single line title_label.setTextFormat(Qt.TextFormat.PlainText) - title_label.setTextInteractionFlags(Qt.TextInteractionFlag.TextSelectableByMouse) # Allow copying + title_label.setTextInteractionFlags( + Qt.TextInteractionFlag.TextSelectableByMouse + ) # Allow copying # Enable elision (text will be cut with ... when too long) from PyQt6.QtWidgets import QSizePolicy - title_label.setSizePolicy(QSizePolicy.Policy.Ignored, QSizePolicy.Policy.Preferred) + + title_label.setSizePolicy( + QSizePolicy.Policy.Ignored, QSizePolicy.Policy.Preferred + ) tab_row.addWidget(title_label, 1) # Stretch to fill available space tab_row.addStretch() @@ -123,7 +158,9 @@ def _setup_ui(self): # Consolidate Results button consolidate_btn = QPushButton("Consolidate Results") consolidate_btn.clicked.connect(self._consolidate_results) - consolidate_btn.setToolTip("Generate MetaXpress-style summary CSV from analysis results") + consolidate_btn.setToolTip( + "Generate MetaXpress-style summary CSV from analysis results" + ) consolidate_btn.setStyleSheet(self.style_gen.generate_button_style()) tab_row.addWidget(consolidate_btn) @@ -135,7 +172,7 @@ def _setup_ui(self): layout.addLayout(tab_row) - # Style the tab bar + # Style tab bar self.tab_bar.setStyleSheet(f""" QTabBar::tab {{ background-color: {self.color_scheme.to_hex(self.color_scheme.input_bg)}; @@ -147,7 +184,7 @@ def _setup_ui(self): border: none; }} QTabBar::tab:selected {{ - background-color: {self.color_scheme.to_hex(self.color_scheme.selection_bg)}; + background-color: {self.color_scheme.to_hex(self.color_scheme.text_accent)}; }} QTabBar::tab:hover {{ background-color: {self.color_scheme.to_hex(self.color_scheme.button_hover_bg)}; @@ -158,13 +195,18 @@ def _setup_ui(self): self.image_browser_tab = self._create_image_browser_tab() self.tab_widget.addTab(self.image_browser_tab, "Image Browser") - # Tab 2: Metadata Viewer - self.metadata_viewer_tab = self._create_metadata_viewer_tab() - self.tab_widget.addTab(self.metadata_viewer_tab, "Metadata") + # Tab 2: Metadata Viewer (lazy-loaded to avoid slow startup) + self.metadata_viewer_tab = self._create_metadata_placeholder_tab() + self._metadata_viewer_loaded = False + self._metadata_tab_index = self.tab_widget.addTab( + self.metadata_viewer_tab, "Metadata" + ) + self.tab_widget.currentChanged.connect(self._on_tab_changed) # Add the tab widget's content area (stacked widget) below the tab row # The tab bar is already in tab_row, so we only add the content pane here from PyQt6.QtWidgets import QStackedWidget + content_container = QWidget() content_layout = QVBoxLayout(content_container) content_layout.setContentsMargins(0, 0, 0, 0) @@ -176,27 +218,53 @@ def _setup_ui(self): content_layout.addWidget(stacked_widget) layout.addWidget(content_container) - + def _create_image_browser_tab(self) -> QWidget: """Create the image browser tab.""" from openhcs.pyqt_gui.widgets.image_browser import ImageBrowserWidget - + # Create image browser widget browser = ImageBrowserWidget( - orchestrator=self.orchestrator, - color_scheme=self.color_scheme, - parent=self + orchestrator=self.orchestrator, color_scheme=self.color_scheme, parent=self ) - + # Store reference self.image_browser = browser - + return browser - + + def _create_metadata_placeholder_tab(self) -> QWidget: + """Create a lightweight placeholder for lazy metadata loading.""" + placeholder = QWidget() + layout = QVBoxLayout(placeholder) + layout.setContentsMargins(10, 10, 10, 10) + label = QLabel("Open this tab to load metadata...") + layout.addWidget(label) + layout.addStretch() + return placeholder + + def _on_tab_changed(self, index: int) -> None: + """Lazy-load metadata viewer when the Metadata tab is opened.""" + if getattr(self, "_metadata_viewer_loaded", False): + return + if index != getattr(self, "_metadata_tab_index", -1): + return + + from PyQt6.QtCore import QSignalBlocker + + self._metadata_viewer_loaded = True + metadata_viewer = self._create_metadata_viewer_tab() + with QSignalBlocker(self.tab_widget): + self.tab_widget.removeTab(index) + self.tab_widget.insertTab(index, metadata_viewer, "Metadata") + self.tab_widget.setCurrentIndex(index) + self.metadata_viewer_tab = metadata_viewer + def _create_metadata_viewer_tab(self) -> QWidget: """Create the metadata viewer tab.""" # Create scroll area for metadata content - from PyQt6.QtWidgets import QScrollArea + from PyQt6.QtWidgets import QScrollArea, QComboBox, QHBoxLayout + scroll_area = QScrollArea() scroll_area.setWidgetResizable(True) scroll_area.setFrameShape(QScrollArea.Shape.NoFrame) @@ -210,44 +278,79 @@ def _create_metadata_viewer_tab(self) -> QWidget: try: metadata_handler = self.orchestrator.microscope_handler.metadata_handler plate_path = self.orchestrator.plate_path - + # Check if this is OpenHCS format - if hasattr(metadata_handler, '_load_metadata_dict'): + if hasattr(metadata_handler, "_load_metadata_dict"): # OpenHCS format from openhcs.microscopes.openhcs import OpenHCSMetadata + metadata_dict = metadata_handler._load_metadata_dict(plate_path) subdirs_dict = metadata_dict.get("subdirectories", {}) if not subdirs_dict: raise ValueError("No subdirectories found in metadata") - # Convert raw dicts to OpenHCSMetadata instances - subdirs_instances = {} - for subdir_name, subdir_data in subdirs_dict.items(): - # Ensure all optional fields have explicit None if missing - # (OpenHCSMetadata requires all fields to be provided, even if Optional) - subdir_data.setdefault('timepoints', None) - subdir_data.setdefault('channels', None) - subdir_data.setdefault('wells', None) - subdir_data.setdefault('sites', None) - subdir_data.setdefault('z_indexes', None) - - # Create OpenHCSMetadata from the subdirectory data - subdirs_instances[subdir_name] = OpenHCSMetadata(**subdir_data) - - # Create forms for each subdirectory - self._create_multi_subdirectory_forms(layout, subdirs_instances) + def ensure_optional_fields(subdir_data): + subdir_data.setdefault("timepoints", None) + subdir_data.setdefault("channels", None) + subdir_data.setdefault("wells", None) + subdir_data.setdefault("sites", None) + subdir_data.setdefault("z_indexes", None) + + if len(subdirs_dict) == 1: + subdir_name = next(iter(subdirs_dict.keys())) + subdir_data = subdirs_dict[subdir_name] + ensure_optional_fields(subdir_data) + metadata_instance = OpenHCSMetadata(**subdir_data) + self._create_single_metadata_form(layout, metadata_instance) + else: + selector_row = QHBoxLayout() + selector_label = QLabel("Subdirectory:") + selector = QComboBox() + selector.addItems(sorted(subdirs_dict.keys())) + selector_row.addWidget(selector_label) + selector_row.addWidget(selector, 1) + layout.addLayout(selector_row) + + form_container = QWidget() + form_layout = QVBoxLayout(form_container) + form_layout.setContentsMargins(0, 0, 0, 0) + layout.addWidget(form_container) + + def clear_layout(target_layout): + while target_layout.count(): + item = target_layout.takeAt(0) + widget = item.widget() + if widget is not None: + widget.deleteLater() + + def render_selected(subdir_name): + clear_layout(form_layout) + subdir_data = subdirs_dict[subdir_name] + ensure_optional_fields(subdir_data) + metadata_instance = OpenHCSMetadata(**subdir_data) + self._create_single_metadata_form( + form_layout, metadata_instance + ) + + selector.currentTextChanged.connect(render_selected) + render_selected(selector.currentText()) else: # Other microscope formats (ImageXpress, Opera Phenix, etc.) from openhcs.microscopes.openhcs import OpenHCSMetadata + component_metadata = metadata_handler.parse_metadata(plate_path) # Get image files list (all handlers have this method) image_files = metadata_handler.get_image_files(plate_path) # Get optional metadata with fallback - grid_dims = metadata_handler._get_with_fallback('get_grid_dimensions', plate_path) - pixel_size = metadata_handler._get_with_fallback('get_pixel_size', plate_path) + grid_dims = metadata_handler._get_with_fallback( + "get_grid_dimensions", plate_path + ) + pixel_size = metadata_handler._get_with_fallback( + "get_pixel_size", plate_path + ) metadata_instance = OpenHCSMetadata( microscope_handler_name=self.orchestrator.microscope_handler.microscope_type, @@ -255,17 +358,17 @@ def _create_metadata_viewer_tab(self) -> QWidget: grid_dimensions=list(grid_dims) if grid_dims else [1, 1], pixel_size=pixel_size if pixel_size else 1.0, image_files=image_files, # Now populated! - channels=component_metadata.get('channel'), - wells=component_metadata.get('well'), - sites=component_metadata.get('site'), - z_indexes=component_metadata.get('z_index'), - timepoints=component_metadata.get('timepoint'), - available_backends={'disk': True}, - main=None + channels=component_metadata.get("channel"), + wells=component_metadata.get("well"), + sites=component_metadata.get("site"), + z_indexes=component_metadata.get("z_index"), + timepoints=component_metadata.get("timepoint"), + available_backends={"disk": True}, + main=None, ) self._create_single_metadata_form(layout, metadata_instance) - + except Exception as e: logger.error(f"Failed to load metadata: {e}", exc_info=True) error_label = QLabel(f"Error loading metadata:
{str(e)}") @@ -278,16 +381,20 @@ def _create_metadata_viewer_tab(self) -> QWidget: # Set container as scroll area widget scroll_area.setWidget(container) return scroll_area - + def _create_single_metadata_form(self, layout, metadata_instance): """Create a single metadata form.""" from pyqt_reactive.forms import ParameterFormManager, FormManagerConfig from openhcs.config_framework.object_state import ObjectState - # Create local ObjectState for metadata viewer + image_files = getattr(metadata_instance, "image_files", None) + if image_files is not None: + layout.addWidget(QLabel(f"Image files: {len(image_files)} (hidden)")) + + # Create local ObjectState for metadata viewer using plate scope for correct accent color state = ObjectState( object_instance=metadata_instance, - scope_id=None, + scope_id=self._style_scope_id, ) metadata_form = ParameterFormManager( @@ -295,8 +402,9 @@ def _create_single_metadata_form(self, layout, metadata_instance): config=FormManagerConfig( parent=None, read_only=True, - color_scheme=self.color_scheme - ) + color_scheme=self.color_scheme, + exclude_params=["image_files", "workspace_mapping"], + ), ) layout.addWidget(metadata_form) @@ -310,10 +418,16 @@ def _create_multi_subdirectory_forms(self, layout, subdirs_instances): group_box = QGroupBox(f"Subdirectory: {subdir_name}") group_layout = QVBoxLayout(group_box) - # Create local ObjectState for this subdirectory's metadata + image_files = getattr(metadata_instance, "image_files", None) + if image_files is not None: + group_layout.addWidget( + QLabel(f"Image files: {len(image_files)} (hidden)") + ) + + # Create local ObjectState for this subdirectory's metadata using plate scope state = ObjectState( object_instance=metadata_instance, - scope_id=None, + scope_id=self._style_scope_id, ) metadata_form = ParameterFormManager( @@ -321,8 +435,9 @@ def _create_multi_subdirectory_forms(self, layout, subdirs_instances): config=FormManagerConfig( parent=None, read_only=True, - color_scheme=self.color_scheme - ) + color_scheme=self.color_scheme, + exclude_params=["image_files", "workspace_mapping"], + ), ) group_layout.addWidget(metadata_form) @@ -340,9 +455,10 @@ def _consolidate_results(self): # Load metadata to get subdirectories from openhcs.microscopes.openhcs import OpenHCSMetadataHandler + if isinstance(metadata_handler, OpenHCSMetadataHandler): metadata_dict = metadata_handler._load_metadata_dict(plate_path) - subdirs = metadata_dict.get('subdirectories', {}) + subdirs = metadata_dict.get("subdirectories", {}) else: # For non-OpenHCS formats, no subdirectories subdirs = {} @@ -352,7 +468,7 @@ def _consolidate_results(self): if subdirs: for subdir_data in subdirs.values(): # Each subdirectory has a results_dir field pointing to its results directory - results_dir_name = subdir_data.get('results_dir') + results_dir_name = subdir_data.get("results_dir") if results_dir_name: results_dir = plate_path / results_dir_name if results_dir.exists() and results_dir.is_dir(): @@ -360,38 +476,50 @@ def _consolidate_results(self): else: # Fallback: scan plate directory for any *_results directories for item in plate_path.iterdir(): - if item.is_dir() and item.name.endswith('_results'): + if item.is_dir() and item.name.endswith("_results"): results_dirs.append(item) if not results_dirs: QMessageBox.warning( self, "No Results Found", - f"No *_results directories found in {plate_path}." + f"No *_results directories found in {plate_path}.", ) return - # Get global config - from openhcs.config_framework.global_config import get_current_global_config - from openhcs.core.config import GlobalPipelineConfig - global_config = get_current_global_config(GlobalPipelineConfig) - - if not global_config: + # Get resolved configs from orchestrator's pipeline_config using ObjectState + if not self.orchestrator.pipeline_config: QMessageBox.warning( self, - "No Global Config", - "No global configuration found. Please ensure the application is properly initialized." + "No Pipeline Config", + "No pipeline configuration found. Please ensure the orchestrator is properly initialized.", ) return + # Create ObjectState from pipeline_config and resolve configs + from openhcs.config_framework.object_state import ObjectState + + pipeline_config_state = ObjectState(self.orchestrator.pipeline_config) + analysis_consolidation_config = ( + pipeline_config_state.get_saved_resolved_value( + "analysis_consolidation_config" + ) + ) + plate_metadata_config = pipeline_config_state.get_saved_resolved_value( + "plate_metadata_config" + ) + # Use consolidated function that handles both per-directory and global consolidation - from openhcs.processing.backends.analysis.consolidate_analysis_results import consolidate_results_directories + from openhcs.processing.backends.analysis.consolidate_analysis_results import ( + consolidate_results_directories, + ) successful_dirs, failed_dirs = consolidate_results_directories( results_dirs=results_dirs, plate_path=plate_path, - global_config=global_config, - filename_parser=self.orchestrator.microscope_handler.parser + analysis_consolidation_config=analysis_consolidation_config, + plate_metadata_config=plate_metadata_config, + filename_parser=self.orchestrator.microscope_handler.parser, ) # Show results @@ -399,31 +527,32 @@ def _consolidate_results(self): QMessageBox.warning( self, "No CSV Files", - f"No CSV files found in any results directories. Nothing to consolidate." + f"No CSV files found in any results directories. Nothing to consolidate.", ) elif successful_dirs and not failed_dirs: - msg = f"Successfully consolidated {len(successful_dirs)} results directories:\n" + "\n".join(f" ✓ {d}" for d in successful_dirs) - if len(successful_dirs) > 1: - msg += f"\n\n✅ Global summary created: {global_config.analysis_consolidation_config.global_summary_filename}" - QMessageBox.information( - self, - "Consolidation Complete", - msg + msg = ( + f"Successfully consolidated {len(successful_dirs)} results directories:\n" + + "\n".join(f" ✓ {d}" for d in successful_dirs) ) + if len(successful_dirs) > 1: + msg += f"\n\n✅ Global summary created: {analysis_consolidation_config.global_summary_filename}" + QMessageBox.information(self, "Consolidation Complete", msg) elif successful_dirs and failed_dirs: QMessageBox.warning( self, "Partial Success", f"Consolidated {len(successful_dirs)} of {len(results_dirs)} directories.\n\n" - f"Successful:\n" + "\n".join(f" ✓ {d}" for d in successful_dirs) + "\n\n" - f"Failed:\n" + "\n".join(f" ✗ {d}: {e}" for d, e in failed_dirs) + f"Successful:\n" + + "\n".join(f" ✓ {d}" for d in successful_dirs) + + "\n\n" + f"Failed:\n" + "\n".join(f" ✗ {d}: {e}" for d, e in failed_dirs), ) else: QMessageBox.critical( self, "Consolidation Failed", - f"All {len(failed_dirs)} directories failed to consolidate:\n\n" + - "\n".join(f" ✗ {d}: {e}" for d, e in failed_dirs) + f"All {len(failed_dirs)} directories failed to consolidate:\n\n" + + "\n".join(f" ✗ {d}: {e}" for d, e in failed_dirs), ) except Exception as e: @@ -431,11 +560,10 @@ def _consolidate_results(self): QMessageBox.critical( self, "Consolidation Failed", - f"Failed to consolidate results:\n\n{str(e)}" + f"Failed to consolidate results:\n\n{str(e)}", ) def cleanup(self): """Clean up resources.""" - if hasattr(self, 'image_browser'): + if hasattr(self, "image_browser"): self.image_browser.cleanup() - diff --git a/openhcs/pyqt_gui/windows/snapshot_browser_window.py b/openhcs/pyqt_gui/windows/snapshot_browser_window.py index 33e6afbc3..01d5da034 100644 --- a/openhcs/pyqt_gui/windows/snapshot_browser_window.py +++ b/openhcs/pyqt_gui/windows/snapshot_browser_window.py @@ -9,13 +9,22 @@ from typing import Dict, List, Optional import datetime -from PyQt6.QtWidgets import QMainWindow, QVBoxLayout, QWidget, QToolBar, QFrame, QComboBox, QLabel +from PyQt6.QtWidgets import ( + QMainWindow, + QVBoxLayout, + QWidget, + QToolBar, + QFrame, + QComboBox, + QLabel, +) from PyQt6.QtCore import Qt from PyQt6.QtGui import QAction from openhcs.config_framework.object_state import ObjectStateRegistry from pyqt_reactive.widgets.shared.abstract_table_browser import ( - AbstractTableBrowser, ColumnDef + AbstractTableBrowser, + ColumnDef, ) from pyqt_reactive.theming import ColorScheme from openhcs.pyqt_gui.widgets.shared.time_travel_widget import TimeTravelWidget @@ -24,6 +33,7 @@ @dataclass class SnapshotInfo: """Snapshot data for display in table.""" + index: int timestamp: str timestamp_raw: float @@ -65,18 +75,22 @@ def on_item_double_clicked(self, key: str, item: SnapshotInfo): def _refresh_from_registry(self): """Refresh table from ObjectStateRegistry history.""" - history = ObjectStateRegistry.get_history_info() + # OpenHCS-specific filter: hide system-only scopes + HIDDEN_SCOPES = {"", "__plates__"} + history = ObjectStateRegistry.get_history_info( + filter_fn=lambda scope_id: scope_id not in HIDDEN_SCOPES + ) items: Dict[str, SnapshotInfo] = {} for h in history: - key = str(h['index']) + key = str(h["index"]) items[key] = SnapshotInfo( - index=h['index'], - timestamp=h['timestamp'], + index=h["index"], + timestamp=h["timestamp"], timestamp_raw=0.0, # Not exposed by get_history_info - label=h['label'], - num_states=h['num_states'], - is_current=h['is_current'], + label=h["label"], + num_states=h["num_states"], + is_current=h["is_current"], ) self.set_items(items) @@ -119,14 +133,14 @@ def __init__(self, color_scheme: Optional[ColorScheme] = None, parent=None): # Timeline widget at bottom (stays in sync with all other timeline widgets) # No browse button - we're already in the browser window self.timeline_widget = TimeTravelWidget( - color_scheme=self.color_scheme, - show_browse_button=False, - parent=self + color_scheme=self.color_scheme, show_browse_button=False, parent=self ) layout.addWidget(self.timeline_widget) # Connect timeline widget position changes to refresh table - self.timeline_widget.position_changed.connect(self._on_timeline_position_changed) + self.timeline_widget.position_changed.connect( + self._on_timeline_position_changed + ) # Subscribe to history changes to refresh table (event-based, no polling) ObjectStateRegistry.add_history_changed_callback(self._on_refresh) @@ -163,7 +177,9 @@ def _setup_toolbar(self): # Delete branch action self.delete_branch_action = QAction("🗑️", self) - self.delete_branch_action.setToolTip("Delete current branch (cannot delete 'main')") + self.delete_branch_action.setToolTip( + "Delete current branch (cannot delete 'main')" + ) self.delete_branch_action.triggered.connect(self._on_delete_branch) toolbar.addAction(self.delete_branch_action) @@ -176,9 +192,9 @@ def _update_branch_ui(self): self.branch_combo.blockSignals(True) self.branch_combo.clear() for b in branches: - self.branch_combo.addItem(b['name']) - if b['is_current']: - self.branch_combo.setCurrentText(b['name']) + self.branch_combo.addItem(b["name"]) + if b["is_current"]: + self.branch_combo.setCurrentText(b["name"]) self.branch_combo.blockSignals(False) # Can't delete main diff --git a/openhcs/pyqt_gui/windows/synthetic_plate_generator_window.py b/openhcs/pyqt_gui/windows/synthetic_plate_generator_window.py index dae3eed6f..961b131d1 100644 --- a/openhcs/pyqt_gui/windows/synthetic_plate_generator_window.py +++ b/openhcs/pyqt_gui/windows/synthetic_plate_generator_window.py @@ -11,16 +11,26 @@ from typing import Optional from PyQt6.QtWidgets import ( - QDialog, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, - QScrollArea, QWidget, QMessageBox, QFileDialog + QDialog, + QVBoxLayout, + QHBoxLayout, + QPushButton, + QLabel, + QScrollArea, + QWidget, + QMessageBox, + QFileDialog, ) from PyQt6.QtCore import Qt, pyqtSignal from PyQt6.QtGui import QFont from pyqt_reactive.forms import ParameterFormManager +from pyqt_reactive.forms.layout_constants import CURRENT_LAYOUT from pyqt_reactive.theming import StyleSheetGenerator from pyqt_reactive.theming import ColorScheme -from openhcs.tests.generators.generate_synthetic_data import SyntheticMicroscopyGenerator +from openhcs.tests.generators.generate_synthetic_data import ( + SyntheticMicroscopyGenerator, +) logger = logging.getLogger(__name__) @@ -28,36 +38,36 @@ class SyntheticPlateGeneratorWindow(QDialog): """ Dialog window for generating synthetic microscopy plates. - + Provides a parameter form for SyntheticMicroscopyGenerator with sensible defaults from the test suite, allowing users to easily generate test data. """ - + # Signals plate_generated = pyqtSignal(str, str) # output_dir path, pipeline_path - + def __init__(self, color_scheme: Optional[ColorScheme] = None, parent=None): """ Initialize the synthetic plate generator window. - + Args: color_scheme: Color scheme for styling (optional) parent: Parent widget """ super().__init__(parent) - + # Initialize color scheme and style generator self.color_scheme = color_scheme or ColorScheme() self.style_generator = StyleSheetGenerator(self.color_scheme) - + # Output directory (will be set by user or use temp) self.output_dir: Optional[str] = None - + # Setup UI self.setup_ui() - + logger.debug("Synthetic plate generator window initialized") - + def setup_ui(self): """Setup the user interface.""" self.setWindowTitle("Generate Synthetic Plate") @@ -67,15 +77,17 @@ def setup_ui(self): layout = QVBoxLayout(self) layout.setSpacing(10) - + # Header with title and description header_widget = QWidget() header_layout = QVBoxLayout(header_widget) header_layout.setContentsMargins(10, 10, 10, 10) - + header_label = QLabel("Generate Synthetic Microscopy Plate") header_label.setFont(QFont("Arial", 14, QFont.Weight.Bold)) - header_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};") + header_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_accent)};" + ) header_layout.addWidget(header_label) description_label = QLabel( @@ -84,15 +96,17 @@ def setup_ui(self): "with 3x3 grid, 2 channels, and 3 z-levels." ) description_label.setWordWrap(True) - description_label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)}; padding: 5px;") + description_label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)}; padding: 5px;" + ) header_layout.addWidget(description_label) - + layout.addWidget(header_widget) - + # Output directory selection output_dir_widget = self._create_output_dir_selector() layout.addWidget(output_dir_widget) - + # Parameter form with scroll area scroll_area = QScrollArea() scroll_area.setWidgetResizable(True) @@ -109,100 +123,107 @@ def setup_ui(self): self.state = ObjectState( object_instance=SyntheticMicroscopyGenerator, # Pass the class itself, not __init__ scope_id=None, - exclude_params=['output_dir', 'skip_files', 'include_all_components', 'random_seed'], + exclude_params=[ + "output_dir", + "skip_files", + "include_all_components", + "random_seed", + ], ) self.form_manager = ParameterFormManager( state=self.state, config=FormManagerConfig( parent=self, - color_scheme=self.color_scheme # Pass color_scheme as instance parameter - ) + color_scheme=self.color_scheme, # Pass color_scheme as instance parameter + ), ) scroll_area.setWidget(self.form_manager) layout.addWidget(scroll_area) - + # Button row button_row = QHBoxLayout() button_row.setContentsMargins(10, 5, 10, 10) button_row.setSpacing(10) - + button_row.addStretch() - + # Get centralized button styles button_styles = self.style_generator.generate_config_button_styles() - + # Cancel button cancel_button = QPushButton("Cancel") - cancel_button.setFixedHeight(28) + cancel_button.setFixedHeight(CURRENT_LAYOUT.button_height) cancel_button.setMinimumWidth(100) cancel_button.clicked.connect(self.reject) cancel_button.setStyleSheet(button_styles["cancel"]) button_row.addWidget(cancel_button) - + # Generate button generate_button = QPushButton("Generate Plate") - generate_button.setFixedHeight(28) + generate_button.setFixedHeight(CURRENT_LAYOUT.button_height) generate_button.setMinimumWidth(100) generate_button.clicked.connect(self.generate_plate) generate_button.setStyleSheet(button_styles["save"]) button_row.addWidget(generate_button) - + layout.addLayout(button_row) # Apply window styling self.setStyleSheet(self.style_generator.generate_dialog_style()) - + def _create_output_dir_selector(self) -> QWidget: """Create the output directory selector widget.""" widget = QWidget() layout = QHBoxLayout(widget) layout.setContentsMargins(10, 5, 10, 5) layout.setSpacing(10) - + label = QLabel("Output Directory:") - label.setStyleSheet(f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)};") + label.setStyleSheet( + f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)};" + ) layout.addWidget(label) - + self.output_dir_label = QLabel("") self.output_dir_label.setStyleSheet( f"color: {self.color_scheme.to_hex(self.color_scheme.text_secondary)}; " f"font-style: italic; padding: 5px;" ) layout.addWidget(self.output_dir_label, 1) - + browse_button = QPushButton("Browse...") - browse_button.setFixedHeight(28) + browse_button.setFixedHeight(CURRENT_LAYOUT.button_height) browse_button.setMinimumWidth(80) browse_button.clicked.connect(self.browse_output_dir) browse_button.setStyleSheet(self.style_generator.generate_button_style()) layout.addWidget(browse_button) - + return widget - + def browse_output_dir(self): """Open file dialog to select output directory.""" dir_path = QFileDialog.getExistingDirectory( self, "Select Output Directory for Synthetic Plate", str(Path.home()), - QFileDialog.Option.ShowDirsOnly + QFileDialog.Option.ShowDirsOnly, ) - + if dir_path: self.output_dir = dir_path self.output_dir_label.setText(dir_path) self.output_dir_label.setStyleSheet( f"color: {self.color_scheme.to_hex(self.color_scheme.text_primary)}; padding: 5px;" ) - + def generate_plate(self): """Generate the synthetic plate with current parameters.""" try: # Get parameters from state params = self.state.get_current_values() - + # Determine output directory if self.output_dir is None: # Use temp directory @@ -211,53 +232,31 @@ def generate_plate(self): logger.info(f"Using temporary directory: {output_dir}") else: output_dir = self.output_dir - + # Add output_dir to params - params['output_dir'] = output_dir - - # Show progress message - QMessageBox.information( - self, - "Generating Plate", - f"Generating synthetic plate at:\n{output_dir}\n\n" - f"This may take a moment...", - QMessageBox.StandardButton.Ok - ) - + params["output_dir"] = output_dir + # Create generator and generate dataset logger.info(f"Generating synthetic plate with params: {params}") generator = SyntheticMicroscopyGenerator(**params) generator.generate_dataset() - + logger.info(f"Successfully generated synthetic plate at: {output_dir}") # Get path to test_pipeline.py from openhcs.tests import test_pipeline + pipeline_path = Path(test_pipeline.__file__) # Emit signal with output directory and pipeline path self.plate_generated.emit(output_dir, str(pipeline_path)) - - # Show success message - QMessageBox.information( - self, - "Success", - f"Synthetic plate generated successfully!\n\nLocation: {output_dir}", - QMessageBox.StandardButton.Ok - ) - + # Close dialog self.accept() - + except Exception as e: logger.error(f"Failed to generate synthetic plate: {e}", exc_info=True) - QMessageBox.critical( - self, - "Generation Failed", - f"Failed to generate synthetic plate:\n\n{str(e)}", - QMessageBox.StandardButton.Ok - ) - + def reject(self): """Handle dialog rejection (Cancel button).""" # Cleanup before closing @@ -273,7 +272,7 @@ def accept(self): def _cleanup(self): """Cleanup resources before window closes.""" # Unregister from cross-window updates - if hasattr(self, 'form_manager') and self.form_manager is not None: + if hasattr(self, "form_manager") and self.form_manager is not None: try: self.form_manager.unregister_from_cross_window_updates() except RuntimeError: @@ -286,4 +285,3 @@ def _cleanup(self): except (RuntimeError, TypeError): # Already disconnected or no connections, ignore pass - diff --git a/openhcs/runtime/execution_server.py b/openhcs/runtime/execution_server.py deleted file mode 100644 index 6d3b8b5d3..000000000 --- a/openhcs/runtime/execution_server.py +++ /dev/null @@ -1,528 +0,0 @@ -#!/usr/bin/env python3 -""" -OpenHCS Execution Server - -Minimal server daemon for remote pipeline execution. -Receives Python code, compiles server-side, executes, streams results. -""" - -import logging -import signal -import time -import uuid -from pathlib import Path -from typing import Any, Dict, Optional - -logger = logging.getLogger(__name__) - - -class OpenHCSExecutionServer: - """Server daemon for remote pipeline execution.""" - - def __init__( - self, - omero_data_dir: Optional[Path] = None, - omero_host: str = 'localhost', - omero_port: int = 4064, - omero_user: str = 'root', - omero_password: str = 'omero-root-password', - server_port: int = None - ): - from openhcs.constants.constants import DEFAULT_EXECUTION_SERVER_PORT - if server_port is None: - server_port = DEFAULT_EXECUTION_SERVER_PORT - self.omero_data_dir = Path(omero_data_dir) if omero_data_dir else None - self.omero_host = omero_host - self.omero_port = omero_port - self.omero_user = omero_user - self.omero_password = omero_password - self.server_port = server_port - - self.running = False - self.omero_conn = None - self.zmq_context = None - self.zmq_socket = None - self.active_executions: Dict[str, Dict[str, Any]] = {} - self.start_time = None - - signal.signal(signal.SIGTERM, lambda s, f: self.shutdown()) - signal.signal(signal.SIGINT, lambda s, f: self.shutdown()) - - def start(self): - """Start server: connect OMERO, register backend, listen for requests.""" - logger.info("Starting OpenHCS Execution Server...") - self.start_time = time.time() - - self._connect_omero() - self._register_backend() - self._setup_zmq() - - self.running = True - - logger.info("=" * 60) - logger.info(f"OpenHCS Execution Server READY on port {self.server_port}") - logger.info("=" * 60) - - self.run() - - def _connect_omero(self): - """Connect to OMERO server.""" - from omero.gateway import BlitzGateway - - self.omero_conn = BlitzGateway( - self.omero_user, - self.omero_password, - host=self.omero_host, - port=self.omero_port - ) - - if not self.omero_conn.connect(): - raise RuntimeError("Failed to connect to OMERO") - - logger.info(f"✓ Connected to OMERO at {self.omero_host}:{self.omero_port}") - - def _register_backend(self): - """Initialize and register OMERO backend.""" - # Import OMERO parsers BEFORE creating backend to ensure registration - # This is required because OMEROLocalBackend accesses FilenameParser.__registry__ - # which is a LazyDiscoveryDict that only populates when first accessed - from openhcs.microscopes import omero # noqa: F401 - Import OMERO parsers to register them - from polystore.omero_local import OMEROLocalBackend - from polystore.backend_registry import _backend_instances - - backend = OMEROLocalBackend( - omero_data_dir=self.omero_data_dir, - omero_conn=self.omero_conn, - namespace_prefix="openhcs", - lock_dir_name=".openhcs", - ) - - _backend_instances['omero_local'] = backend - logger.info("✓ OMERO backend registered") - - def _setup_zmq(self): - """Set up ZeroMQ socket.""" - import zmq - - self.zmq_context = zmq.Context() - self.zmq_socket = self.zmq_context.socket(zmq.REP) - self.zmq_socket.bind(f"tcp://*:{self.server_port}") - - logger.info(f"✓ Listening on tcp://*:{self.server_port}") - - def run(self): - """Main server loop.""" - while self.running: - # CRITICAL: ZMQ REP sockets require strict recv->send->recv->send alternation - # If recv() succeeds but send() doesn't happen, the socket enters an invalid - # state and refuses all future recv() calls, causing the server to hang. - - # Step 1: Try to receive a message (blocking) - try: - message = self.zmq_socket.recv_json() - except KeyboardInterrupt: - break - except Exception as e: - # recv failed - no message received, so no need to send response - logger.error(f"Error receiving message: {e}", exc_info=True) - continue - - # Step 2: We received a message, so we MUST send a response - try: - response = self._handle_request(message) - except Exception as e: - # ANY error during processing - send error response to maintain socket state - logger.error(f"Error handling request: {e}", exc_info=True) - response = {'status': 'error', 'message': str(e)} - - # Step 3: ALWAYS send response (even if it's an error response) - try: - self.zmq_socket.send_json(response) - except Exception as e: - # If send fails, the socket is likely broken - log and continue - logger.error(f"Failed to send response: {e}", exc_info=True) - - def _handle_request(self, msg: Dict[str, Any]) -> Dict[str, Any]: - """Route request to appropriate handler.""" - handlers = { - 'execute': self._handle_execute, - 'status': self._handle_status, - 'ping': lambda m: {'status': 'ok', 'message': 'pong'} - } - - handler = handlers.get(msg.get('command')) - if not handler: - return {'status': 'error', 'message': f"Unknown command: {msg.get('command')}"} - - return handler(msg) - - def _handle_execute(self, msg: Dict[str, Any]) -> Dict[str, Any]: - """Handle execution request - executes synchronously in main thread.""" - # Require plate_id and pipeline_code - # Accept EITHER config_params (dict) OR config_code (Python code) - # Optionally accept pipeline_config_code for separate PipelineConfig - # Optionally accept omero_config for OMERO mode - if 'plate_id' not in msg or 'pipeline_code' not in msg: - return {'status': 'error', 'message': 'Missing required fields: plate_id, pipeline_code'} - - if 'config_params' not in msg and 'config_code' not in msg: - return {'status': 'error', 'message': 'Missing config: provide either config_params or config_code'} - - execution_id = str(uuid.uuid4()) - - record = { - 'execution_id': execution_id, - 'plate_id': msg['plate_id'], - 'client_address': msg.get('client_address'), - 'status': 'running', - 'start_time': time.time(), - 'end_time': None, - 'error': None - } - - self.active_executions[execution_id] = record - - # Execute synchronously in main thread (like UI does) - # This ensures exec() runs in main thread, not worker thread - try: - results = self._execute_pipeline( - execution_id, - msg['plate_id'], - msg['pipeline_code'], - msg.get('config_params'), # May be None - msg.get('config_code'), # May be None - msg.get('pipeline_config_code'), # May be None - separate PipelineConfig code - msg.get('client_address'), - msg.get('omero_config') # May be None - OMERO connection config - ) - record['status'] = 'complete' - record['end_time'] = time.time() - record['results'] = results - - return { - 'status': 'complete', - 'execution_id': execution_id, - 'results': results - } - except Exception as e: - record['status'] = 'failed' - record['end_time'] = time.time() - record['error'] = str(e) - logger.error(f"[{execution_id}] ✗ Failed: {e}") - - return { - 'status': 'error', - 'execution_id': execution_id, - 'message': str(e) - } - - def _handle_status(self, msg: Dict[str, Any]) -> Dict[str, Any]: - """Handle status request.""" - if execution_id := msg.get('execution_id'): - if execution_id not in self.active_executions: - return {'status': 'error', 'message': f"Execution not found: {execution_id}"} - - r = self.active_executions[execution_id] - return { - 'status': 'ok', - 'execution_id': execution_id, - 'execution_status': r['status'], - 'plate_id': r['plate_id'], - 'start_time': r['start_time'], - 'end_time': r['end_time'], - 'error': r['error'] - } - - # Server status - return { - 'status': 'ok', - 'uptime': time.time() - self.start_time if self.start_time else 0, - 'active_executions': sum(1 for r in self.active_executions.values() if r['status'] == 'running'), - 'total_executions': len(self.active_executions), - 'omero_connected': self.omero_conn and self.omero_conn.isConnected() - } - - def _setup_omero_connection(self, omero_config: dict) -> None: - """ - Connect to OMERO and set connection on backend. - - Args: - omero_config: {"host": ..., "port": ..., "username": ..., "password": ...} - """ - from omero.gateway import BlitzGateway - from polystore.base import storage_registry - from openhcs.constants.constants import Backend - - # Connect to OMERO - self.omero_conn = BlitzGateway( - omero_config['username'], - omero_config['password'], - host=omero_config['host'], - port=omero_config.get('port', 4064) - ) - - if not self.omero_conn.connect(): - raise ConnectionError("Failed to connect to OMERO") - - # Set connection on OMERO backend - omero_backend = storage_registry[Backend.OMERO_LOCAL.value] - omero_backend.omero_conn = self.omero_conn - - logger.info(f"Connected to OMERO at {omero_config['host']}") - - def _teardown_omero_connection(self) -> None: - """Close OMERO connection if open.""" - if self.omero_conn: - self.omero_conn.close() - self.omero_conn = None - logger.info("Closed OMERO connection") - - def _execute_pipeline( - self, - execution_id: str, - plate_id: int, - pipeline_code: str, - config_params: Optional[dict], - config_code: Optional[str], - pipeline_config_code: Optional[str], - client_address: Optional[str] = None, - omero_config: Optional[dict] = None - ): - """Execute pipeline: reconstruct from code, compile server-side, execute.""" - record = self.active_executions[execution_id] - record['status'] = 'running' - - try: - # OMERO-specific setup - if omero_config: - self._setup_omero_connection(omero_config) - - logger.info(f"[{execution_id}] Starting execution for plate {plate_id}") - - # Initialize function registry BEFORE executing pipeline code - # (pipeline code may import virtual modules like openhcs.cucim) - import openhcs.processing.func_registry as func_registry_module - with func_registry_module._registry_lock: - if not func_registry_module._registry_initialized: - func_registry_module._auto_initialize_registry() - - # Reconstruct pipeline by executing the exact generated Python code (same as UI) - # Use an empty namespace so imports resolve naturally to module-level symbols - namespace: Dict[str, Any] = {} - exec(pipeline_code, namespace) - pipeline_steps = namespace.get('pipeline_steps') - if not pipeline_steps: - raise ValueError("Code must define 'pipeline_steps'") - - logger.info(f"[{execution_id}] Loaded {len(pipeline_steps)} steps") - - # Create config - support both approaches - if config_code: - # Approach 1: Execute config code to get GlobalPipelineConfig object - logger.info(f"[{execution_id}] Loading config from code...") - # Use same namespace pattern to ensure enum identity - from openhcs.core.config import GlobalPipelineConfig, PipelineConfig - config_namespace = {} - exec(config_code, config_namespace) - - global_config = config_namespace.get('config') - if not global_config: - raise ValueError("config_code must define 'config' variable") - - # Handle PipelineConfig - either from separate code or use defaults - if pipeline_config_code: - logger.info(f"[{execution_id}] Loading PipelineConfig from code...") - pipeline_config_namespace = {} - exec(pipeline_config_code, pipeline_config_namespace) - pipeline_config = pipeline_config_namespace.get('config') - if not pipeline_config: - raise ValueError("pipeline_config_code must define 'config' variable") - else: - # Use defaults - from openhcs.core.config import PipelineConfig - pipeline_config = PipelineConfig() - - elif config_params: - # Approach 2: Build GlobalPipelineConfig/PipelineConfig directly from params - logger.info(f"[{execution_id}] Creating config from params...") - from openhcs.core.config import ( - GlobalPipelineConfig, - MaterializationBackend, - PathPlanningConfig, - StepWellFilterConfig, - VFSConfig, - ZarrConfig, - PipelineConfig, - ) - from openhcs.constants.constants import Backend - - # OMERO mode overrides - if omero_config: - config_params['read_backend'] = Backend.OMERO_LOCAL.value - config_params['materialization_backend'] = 'zarr' # Force zarr for OMERO - - global_config = GlobalPipelineConfig( - num_workers=config_params.get('num_workers', 4), - path_planning_config=PathPlanningConfig( - output_dir_suffix=config_params.get('output_dir_suffix', '_output') - ), - vfs_config=VFSConfig( - read_backend=Backend(config_params.get('read_backend', 'auto')), - materialization_backend=MaterializationBackend( - config_params.get('materialization_backend', 'disk') - ) - ), - zarr_config=ZarrConfig(), - step_well_filter_config=StepWellFilterConfig( - well_filter=config_params.get('well_filter') - ), - use_threading=config_params.get('use_threading', False), - ) - pipeline_config = PipelineConfig() - else: - raise ValueError("Either config_params or config_code must be provided") - - # Update streaming configs to point to client - if client_address: - pipeline_steps = self._update_streaming_configs(pipeline_steps, client_address) - - # Set up orchestrator exactly like test_main.py (no special transport) - from pathlib import Path - from openhcs.config_framework.global_config import ensure_global_config_context - from openhcs.core.orchestrator.gpu_scheduler import setup_global_gpu_registry - from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator - from openhcs.constants import MULTIPROCESSING_AXIS - from polystore.base import reset_memory_backend - - # Reset ephemeral backends and initialize GPU registry - reset_memory_backend() - setup_global_gpu_registry() - - # Install global config context for dual-axis resolver - ensure_global_config_context(type(global_config), global_config) - - orchestrator = PipelineOrchestrator( - plate_path=Path(f"/omero/plate_{plate_id}"), - pipeline_config=pipeline_config - ) - orchestrator.initialize() - - # Execute using standard compile→execute phases - logger.info(f"[{execution_id}] Executing pipeline...") - - # CRITICAL: Compiler now resolves well_filter_config from pipeline_config automatically - # No need to extract it manually here - just pass None to compile_pipelines - # The compiler will check orchestrator.pipeline_config.well_filter_config and use it - compilation = orchestrator.compile_pipelines( - pipeline_definition=pipeline_steps, - well_filter=None, - ) - compiled_contexts = compilation['compiled_contexts'] - - results = orchestrator.execute_compiled_plate( - pipeline_definition=pipeline_steps, - compiled_contexts=compiled_contexts, - ) - - # Mark completed - record['status'] = 'completed' - record['end_time'] = time.time() - record['wells_processed'] = len(results.get('well_results', {})) - - elapsed = record['end_time'] - record['start_time'] - logger.info(f"[{execution_id}] ✓ Completed in {elapsed:.1f}s") - return results - - except Exception as e: - record['status'] = 'error' - record['end_time'] = time.time() - record['error'] = str(e) - logger.error(f"[{execution_id}] ✗ Failed: {e}", exc_info=True) - raise - - finally: - # OMERO-specific teardown - if omero_config: - self._teardown_omero_connection() - - def _update_streaming_configs(self, pipeline_steps, client_address): - """Update streaming configs to point to client.""" - from dataclasses import replace - - client_host, _, port = client_address.rpartition(':') - client_port = int(port) if port else 5555 - - updated_steps = [] - for step in pipeline_steps: - if hasattr(step, 'napari_streaming_config') and step.napari_streaming_config: - new_config = replace( - step.napari_streaming_config, - host=client_host, - port=client_port - ) - step = replace(step, napari_streaming_config=new_config) - updated_steps.append(step) - - return updated_steps - - def shutdown(self): - """Shutdown server gracefully.""" - logger.info("Shutting down...") - self.running = False - - # Clean up resources - if self.omero_conn: - self.omero_conn.close() - if self.zmq_socket: - self.zmq_socket.close() - if self.zmq_context: - self.zmq_context.term() - - logger.info("Shutdown complete") - - -def main(): - """Entry point for execution server.""" - import argparse - from openhcs.constants.constants import DEFAULT_EXECUTION_SERVER_PORT - - parser = argparse.ArgumentParser(description='OpenHCS Execution Server') - parser.add_argument('--omero-data-dir', type=Path, default=None, - help='Path to OMERO binary repository (optional, uses API if not set)') - parser.add_argument('--omero-host', default='localhost') - parser.add_argument('--omero-port', type=int, default=4064) - parser.add_argument('--omero-user', default='root') - parser.add_argument('--omero-password', default='omero-root-password') - parser.add_argument('--port', type=int, default=DEFAULT_EXECUTION_SERVER_PORT) - parser.add_argument('--log-file', type=Path) - parser.add_argument('--log-level', default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR']) - - args = parser.parse_args() - - # Set up logging - logging.basicConfig( - level=getattr(logging, args.log_level), - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - filename=args.log_file - ) - - # Create and start server - server = OpenHCSExecutionServer( - omero_data_dir=args.omero_data_dir, - omero_host=args.omero_host, - omero_port=args.omero_port, - omero_user=args.omero_user, - omero_password=args.omero_password, - server_port=args.port - ) - - try: - server.start() - except KeyboardInterrupt: - pass - finally: - server.shutdown() - - -if __name__ == '__main__': - main() diff --git a/openhcs/runtime/fiji_stream_visualizer.py b/openhcs/runtime/fiji_stream_visualizer.py index 7a9bb6e06..3966e7af7 100644 --- a/openhcs/runtime/fiji_stream_visualizer.py +++ b/openhcs/runtime/fiji_stream_visualizer.py @@ -19,7 +19,10 @@ from polystore.filemanager import FileManager from polystore.backend_registry import register_cleanup_callback -from openhcs.core.config import TransportMode as OpenHCSTransportMode, FijiStreamingConfig +from openhcs.core.config import ( + TransportMode as OpenHCSTransportMode, + FijiStreamingConfig, +) from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG from zmqruntime.config import TransportMode as ZMQTransportMode from zmqruntime.streaming import VisualizerProcessManager @@ -98,8 +101,9 @@ def _spawn_detached_fiji_process( try: from openhcs.runtime.fiji_viewer_server import _fiji_viewer_server_process from openhcs.core.config import TransportMode + from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG transport_mode = TransportMode.{transport_mode.name} - _fiji_viewer_server_process({port}, {repr(viewer_title)}, None, {repr(current_dir + "/.fiji_log_path_placeholder")}, transport_mode) + _fiji_viewer_server_process({port}, {repr(viewer_title)}, None, {repr(current_dir + "/.fiji_log_path_placeholder")}, transport_mode, OPENHCS_ZMQ_CONFIG) except Exception as e: import logging logger = logging.getLogger("openhcs.runtime.fiji_detached") @@ -119,6 +123,12 @@ def _spawn_detached_fiji_process( python_code = python_code.replace( repr(current_dir + "/.fiji_log_path_placeholder"), repr(log_file) ) + # Remove incidental indentation and leading/trailing whitespace from the + # embedded snippet so it runs with the expected top-level indentation when + # passed to `python -c`. + import textwrap + + python_code = textwrap.dedent(python_code).strip() # Use subprocess.Popen with detachment flags if sys.platform == "win32": @@ -166,7 +176,6 @@ class FijiStreamVisualizer(VisualizerProcessManager): """ Manages Fiji viewer instance for real-time visualization via ZMQ. - Uses FijiViewerServer (inherits from ZMQServer) for PyImageJ-based display. Follows same architecture as NapariStreamVisualizer. """ @@ -192,7 +201,9 @@ def __init__( ) super().__init__(port=self.port) self.display_config = display_config - self.transport_mode = coerce_transport_mode(transport_mode) or ZMQTransportMode.IPC # ZMQ transport mode (IPC or TCP) + self.transport_mode = ( + coerce_transport_mode(transport_mode) or ZMQTransportMode.IPC + ) # ZMQ transport mode (IPC or TCP) self.process: Optional[multiprocessing.Process] = None self._is_running = False self._connected_to_existing = False @@ -278,7 +289,9 @@ def start_viewer(self, async_mode: bool = False) -> None: with self._lock: # Check if there's already a viewer running on the configured port - if is_port_in_use(self.port, self.transport_mode, config=OPENHCS_ZMQ_CONFIG): + if is_port_in_use( + self.port, self.transport_mode, config=OPENHCS_ZMQ_CONFIG + ): # Try to connect to existing viewer first logger.info( f"🔬 FIJI VISUALIZER: Port {self.port} is in use, attempting to connect to existing viewer..." @@ -539,10 +552,16 @@ def get_launch_command(self) -> list[str]: sys.path.insert(0, {repr(current_dir)}) from openhcs.runtime.fiji_viewer_server import _fiji_viewer_server_process from openhcs.core.config import TransportMode +from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG transport_mode = TransportMode.{self.transport_mode.name} -_fiji_viewer_server_process({self.port}, {repr(self.viewer_title)}, None, {repr(log_file)}, transport_mode) +_fiji_viewer_server_process({self.port}, {repr(self.viewer_title)}, None, {repr(log_file)}, transport_mode, OPENHCS_ZMQ_CONFIG) """ + # Ensure snippet has no incidental indentation + import textwrap + + python_code = textwrap.dedent(python_code).strip() + return [sys.executable, "-c", python_code] def get_launch_env(self) -> dict: diff --git a/openhcs/runtime/fiji_viewer_server.py b/openhcs/runtime/fiji_viewer_server.py index d508da4cd..44c4b2f6d 100644 --- a/openhcs/runtime/fiji_viewer_server.py +++ b/openhcs/runtime/fiji_viewer_server.py @@ -10,7 +10,11 @@ import threading from typing import Dict, Any, List -from openhcs.constants.streaming import StreamingDataType +from polystore.streaming_constants import StreamingDataType +from polystore.streaming.receivers.core import ( + DebouncedBatchEngine, + group_items_by_component_modes, +) from openhcs.core.config import TransportMode as OpenHCSTransportMode from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG from zmqruntime.transport import coerce_transport_mode @@ -22,11 +26,14 @@ # Registry mapping data types to handler methods _FIJI_ITEM_HANDLERS = {} + def register_fiji_handler(data_type: StreamingDataType): """Decorator to register handler for a data type.""" + def decorator(func): _FIJI_ITEM_HANDLERS[data_type] = func return func + return decorator @@ -39,13 +46,23 @@ class FijiViewerServer(StreamingVisualizerServer): Displays images via PyImageJ. """ - _server_type = 'fiji' # Registration key for AutoRegisterMeta + _server_type = "fiji" # Registration key for AutoRegisterMeta # Debouncing configuration DEBOUNCE_DELAY_MS = 500 # Collect items for 500ms before processing - MAX_DEBOUNCE_WAIT_MS = 2000 # Maximum wait time before forcing batch processing + MAX_DEBOUNCE_WAIT_MS = ( + 2000 # Maximum wait time before forcing batch processing (2s) + ) - def __init__(self, port: int, viewer_title: str, display_config, log_file_path: str = None, transport_mode: OpenHCSTransportMode = OpenHCSTransportMode.IPC): + def __init__( + self, + port: int, + viewer_title: str, + display_config, + log_file_path: str = None, + transport_mode: OpenHCSTransportMode = OpenHCSTransportMode.IPC, + zmq_config=None, + ): """ Initialize Fiji viewer server. @@ -55,11 +72,16 @@ def __init__(self, port: int, viewer_title: str, display_config, log_file_path: display_config: FijiDisplayConfig with LUT, dimension modes, etc. log_file_path: Path to log file (for client discovery) transport_mode: ZMQ transport mode (IPC or TCP) + zmq_config: ZMQ configuration object (optional, uses default if None) """ import zmq # Initialize with REP socket for receiving images (synchronous request/reply) # REP socket forces workers to wait for acknowledgment before closing shared memory + if zmq_config is None: + from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG + + zmq_config = OPENHCS_ZMQ_CONFIG super().__init__( port, viewer_type="fiji", @@ -67,7 +89,7 @@ def __init__(self, port: int, viewer_title: str, display_config, log_file_path: log_file_path=log_file_path, data_socket_type=zmq.REP, transport_mode=coerce_transport_mode(transport_mode), - config=OPENHCS_ZMQ_CONFIG, + config=zmq_config, ) self.viewer_title = viewer_title @@ -79,19 +101,16 @@ def __init__(self, port: int, viewer_title: str, display_config, log_file_path: self.window_key_to_group_id = {} # Map window_key strings to integer group IDs self._next_group_id = 1 # Counter for assigning group IDs self.window_dimension_values = {} # Store dimension values (channel/slice/frame) per window + self.window_fixed_labels = {} # Store fixed window-component labels per window # Ack socket handled by StreamingVisualizerServer - # Debouncing for batched processing - self._pending_items = [] # Queue of copied items waiting to be processed - self._pending_display_config = None # Display config for pending batch - self._pending_images_dir = None # Images dir for pending batch - self._pending_component_names_metadata = {} # Component names metadata for dimension labels - self._debounce_timer = None # Timer for debounced processing - self._debounce_lock = threading.Lock() # Lock for pending queue - self._debounce_delay = self.DEBOUNCE_DELAY_MS / 1000.0 # Convert ms to seconds - self._max_debounce_wait = self.MAX_DEBOUNCE_WAIT_MS / 1000.0 # Convert ms to seconds - self._first_message_time = None # Track when first message in batch arrived + # Debouncing for batched processing (shared core) + self._batch_engine = DebouncedBatchEngine( + process_fn=self._process_items_from_batch_with_context, + debounce_delay_ms=self.DEBOUNCE_DELAY_MS, + max_debounce_wait_ms=self.MAX_DEBOUNCE_WAIT_MS, + ) # Lock for hyperstack operations to prevent concurrent batch processing from # overwriting each other's hyperstacks (race condition in fast sequential mode) @@ -101,7 +120,7 @@ def _setup_ack_socket(self): """Setup PUSH socket for sending acknowledgments.""" super()._setup_ack_socket() - def _send_ack(self, image_id: str, status: str = 'success', error: str = None): + def _send_ack(self, image_id: str, status: str = "success", error: str = None): """Send acknowledgment that an image was processed. Args: @@ -110,7 +129,7 @@ def _send_ack(self, image_id: str, status: str = 'success', error: str = None): error: Error message if status='error' """ self.send_ack(image_id, status=status, error=error) - + def _wait_for_swing_ui_ready(self, timeout: float = 5.0) -> bool: """Wait for Java Swing UI to be fully initialized. @@ -131,7 +150,7 @@ def _wait_for_swing_ui_ready(self, timeout: float = 5.0) -> bool: try: # Try to access UIManager and verify UIDefaults are populated # This is critical because RoiManager needs JList UI components - UIManager = sj.jimport('javax.swing.UIManager') + UIManager = sj.jimport("javax.swing.UIManager") look_and_feel = UIManager.getLookAndFeel() if look_and_feel is not None: @@ -141,10 +160,14 @@ def _wait_for_swing_ui_ready(self, timeout: float = 5.0) -> bool: list_ui = ui_defaults.get("ListUI") if list_ui is not None: - logger.info("🔬 FIJI SERVER: Java Swing UI is ready (UIDefaults populated)") + logger.info( + "🔬 FIJI SERVER: Java Swing UI is ready (UIDefaults populated)" + ) return True else: - logger.debug("🔬 FIJI SERVER: Waiting for UIDefaults to populate...") + logger.debug( + "🔬 FIJI SERVER: Waiting for UIDefaults to populate..." + ) except Exception as e: logger.debug(f"🔬 FIJI SERVER: Waiting for Swing UI: {e}") @@ -160,45 +183,55 @@ def start(self): # Initialize PyImageJ in this process try: import imagej + logger.info("🔬 FIJI SERVER: Initializing PyImageJ...") # Try interactive mode first, fall back to headless mode on macOS try: - self.ij = imagej.init(mode='interactive') + self.ij = imagej.init(mode="interactive") # Show Fiji UI so users can interact with images and menus self.ij.ui().showUI() - logger.info("🔬 FIJI SERVER: PyImageJ initialized in interactive mode with UI shown") + logger.info( + "🔬 FIJI SERVER: PyImageJ initialized in interactive mode with UI shown" + ) # Wait for Java Swing UI to be fully initialized # This is critical for IPC mode where messages arrive very fast # RoiManager creation requires the Swing event dispatch thread to be ready if not self._wait_for_swing_ui_ready(timeout=5.0): - logger.warning("🔬 FIJI SERVER: Swing UI may not be fully initialized, proceeding anyway") + logger.warning( + "🔬 FIJI SERVER: Swing UI may not be fully initialized, proceeding anyway" + ) except OSError as e: if "Cannot enable interactive mode" in str(e): - logger.warning("🔬 FIJI SERVER: Interactive mode failed (likely macOS), using headless mode") - self.ij = imagej.init(mode='headless') + logger.warning( + "🔬 FIJI SERVER: Interactive mode failed (likely macOS), using headless mode" + ) + self.ij = imagej.init(mode="headless") logger.info("🔬 FIJI SERVER: PyImageJ initialized in headless mode") else: raise except ImportError: - raise ImportError("PyImageJ not available. Install with: pip install 'openhcs[viz]'") - + raise ImportError( + "PyImageJ not available. Install with: pip install 'openhcs[viz]'" + ) + def _create_pong_response(self) -> Dict[str, Any]: """Override to add Fiji-specific fields and memory usage.""" response = super()._create_pong_response() - response['viewer'] = 'fiji' - response['openhcs'] = True - response['server'] = 'FijiViewerServer' + response["viewer"] = "fiji" + response["openhcs"] = True + response["server"] = "FijiViewerServer" # Add memory usage try: import psutil import os + process = psutil.Process(os.getpid()) - response['memory_mb'] = process.memory_info().rss / 1024 / 1024 - response['cpu_percent'] = process.cpu_percent(interval=0) + response["memory_mb"] = process.memory_info().rss / 1024 / 1024 + response["cpu_percent"] = process.cpu_percent(interval=0) except Exception: pass @@ -213,37 +246,42 @@ def handle_control_message(self, message: Dict[str, Any]) -> Dict[str, Any]: - force_shutdown: Force shutdown (same as shutdown for Fiji) - clear_state: Clear accumulated dimension values and hyperstack metadata (for new pipeline runs) """ - msg_type = message.get('type') + msg_type = message.get("type") - if msg_type == 'shutdown' or msg_type == 'force_shutdown': - logger.info(f"🔬 FIJI SERVER: {msg_type} requested, will close after sending acknowledgment") + if msg_type == "shutdown" or msg_type == "force_shutdown": + logger.info( + f"🔬 FIJI SERVER: {msg_type} requested, will close after sending acknowledgment" + ) # Set shutdown flag but don't call stop() yet - let the response be sent first self._shutdown_requested = True return { - 'type': 'shutdown_ack', - 'status': 'success', - 'message': 'Fiji viewer shutting down' + "type": "shutdown_ack", + "status": "success", + "message": "Fiji viewer shutting down", } - elif msg_type == 'clear_state': + elif msg_type == "clear_state": # Clear accumulated dimension values to prevent accumulation across runs # Keep hyperstacks and metadata so images at same coordinates get replaced (not accumulated) - logger.info(f"🔬 FIJI SERVER: Clearing dimension values (had {len(self.window_dimension_values)} windows)") + logger.info( + f"🔬 FIJI SERVER: Clearing dimension values (had {len(self.window_dimension_values)} windows)" + ) # Clear only dimension values - this prevents dimension accumulation # while allowing image replacement at same coordinates self.window_dimension_values.clear() + self.window_fixed_labels.clear() # Note: self.hyperstacks and self.hyperstack_metadata are NOT cleared # This allows the rebuild logic to replace images at same CZT coordinates return { - 'type': 'clear_state_ack', - 'status': 'success', - 'message': 'Dimension values cleared' + "type": "clear_state_ack", + "status": "success", + "message": "Dimension values cleared", } - return {'status': 'ok'} - + return {"status": "ok"} + def handle_data_message(self, message: Dict[str, Any]): """Handle incoming image data - called by process_messages().""" pass @@ -252,7 +290,9 @@ def display_image(self, image_data, metadata: dict) -> None: """Display a single image payload (no-op; Fiji uses batch processing).""" return - def _copy_items_from_shared_memory(self, items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def _copy_items_from_shared_memory( + self, items: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: """ Copy data from shared memory to local memory for all items. @@ -272,10 +312,10 @@ def _copy_items_from_shared_memory(self, items: List[Dict[str, Any]]) -> List[Di copied_item = item.copy() # Only copy shared memory for image items (not ROIs) - if item.get('data_type') == 'image' and 'shm_name' in item: - shm_name = item['shm_name'] - shape = tuple(item['shape']) - dtype = np.dtype(item['dtype']) + if item.get("data_type") == "image" and "shm_name" in item: + shm_name = item["shm_name"] + shape = tuple(item["shape"]) + dtype = np.dtype(item["dtype"]) try: # Attach to shared memory and copy data @@ -285,26 +325,36 @@ def _copy_items_from_shared_memory(self, items: List[Dict[str, Any]]) -> List[Di shm.unlink() # Clean up shared memory # Replace shared memory reference with actual data - copied_item['data'] = np_data - del copied_item['shm_name'] # No longer needed - del copied_item['shape'] # No longer needed - del copied_item['dtype'] # No longer needed + copied_item["data"] = np_data + del copied_item["shm_name"] # No longer needed + del copied_item["shape"] # No longer needed + del copied_item["dtype"] # No longer needed - logger.debug(f"📋 FIJI SERVER: Copied image data from shared memory {shm_name}") + logger.debug( + f"📋 FIJI SERVER: Copied image data from shared memory {shm_name}" + ) except Exception as e: - logger.error(f"📋 FIJI SERVER: Failed to copy from shared memory {shm_name}: {e}") + logger.error( + f"📋 FIJI SERVER: Failed to copy from shared memory {shm_name}: {e}" + ) # Send error ack for this image - image_id = item.get('image_id') + image_id = item.get("image_id") if image_id: - self._send_ack(image_id, status='error', error=str(e)) + self._send_ack(image_id, status="error", error=str(e)) continue copied_items.append(copied_item) return copied_items - def _queue_for_debounced_processing(self, items: List[Dict[str, Any]], display_config_dict: Dict[str, Any], images_dir: str, component_names_metadata: Dict[str, Any] = None): + def _queue_for_debounced_processing( + self, + items: List[Dict[str, Any]], + display_config_dict: Dict[str, Any], + images_dir: str, + component_names_metadata: Dict[str, Any] = None, + ): """ Queue items for debounced batch processing. @@ -317,60 +367,35 @@ def _queue_for_debounced_processing(self, items: List[Dict[str, Any]], display_c images_dir: Source image subdirectory component_names_metadata: Component name mappings for dimension labels """ - with self._debounce_lock: - # Add items to pending queue - self._pending_items.extend(items) - self._pending_display_config = display_config_dict - self._pending_images_dir = images_dir - self._pending_component_names_metadata = component_names_metadata or {} - - # Track first message time for max wait enforcement - if self._first_message_time is None: - self._first_message_time = time.time() - - # Cancel existing timer if any - if self._debounce_timer is not None: - self._debounce_timer.cancel() - - # Check if we've exceeded max wait time - elapsed = time.time() - self._first_message_time - if elapsed >= self._max_debounce_wait: - # Process immediately, don't wait any longer - logger.info(f"⏱️ FIJI SERVER: Max debounce wait ({self._max_debounce_wait}s) exceeded, processing {len(self._pending_items)} items immediately") - self._process_pending_batch() - else: - # Start new debounce timer - remaining_wait = min(self._debounce_delay, self._max_debounce_wait - elapsed) - logger.debug(f"⏱️ FIJI SERVER: Debouncing {len(self._pending_items)} items for {remaining_wait:.3f}s") - self._debounce_timer = threading.Timer(remaining_wait, self._process_pending_batch) - self._debounce_timer.start() + self._batch_engine.enqueue( + items=items, + context={ + "display_config_dict": display_config_dict, + "images_dir": images_dir, + "component_names_metadata": component_names_metadata or {}, + }, + ) def _process_pending_batch(self): - """Process all pending items as a batch (called by debounce timer).""" - with self._debounce_lock: - if not self._pending_items: - return - - items = self._pending_items - display_config_dict = self._pending_display_config - images_dir = self._pending_images_dir - component_names_metadata = self._pending_component_names_metadata - - # Clear pending queue - self._pending_items = [] - self._pending_display_config = None - self._pending_images_dir = None - self._pending_component_names_metadata = {} - self._debounce_timer = None - self._first_message_time = None - - logger.info(f"🔄 FIJI SERVER: Processing debounced batch of {len(items)} items") - - # Process batch (outside lock to avoid blocking new messages) - try: - self._process_items_from_batch(items, display_config_dict, images_dir, component_names_metadata) - except Exception as e: - logger.error(f"🔄 FIJI SERVER: Error processing debounced batch: {e}", exc_info=True) + """Process any pending items immediately.""" + self._batch_engine.flush() + + def _process_items_from_batch_with_context( + self, items: List[Dict[str, Any]], context: Dict[str, Any] + ) -> None: + """Batch-engine callback that unpacks context into canonical arguments.""" + if not items: + return + display_config_dict = context["display_config_dict"] + images_dir = context.get("images_dir") + component_names_metadata = context.get("component_names_metadata", {}) + logger.info(f"🔄 FIJI SERVER: Processing debounced batch of {len(items)} items") + self._process_items_from_batch( + items, + display_config_dict, + images_dir, + component_names_metadata, + ) def process_image_message(self, message: bytes) -> dict: """ @@ -389,45 +414,61 @@ def process_image_message(self, message: bytes) -> dict: try: # Parse JSON message - data = json.loads(message.decode('utf-8')) + data = json.loads(message.decode("utf-8")) - msg_type = data.get('type') + msg_type = data.get("type") # Check message type - if msg_type == 'batch': - items = data.get('images', []) - display_config_dict = data.get('display_config', {}) - images_dir = data.get('images_dir') - component_names_metadata = data.get('component_names_metadata', {}) - - logger.info(f"📨 FIJI SERVER: Received batch message with {len(items)} items") + if msg_type == "batch": + items = data.get("images", []) + display_config_dict = data.get("display_config", {}) + images_dir = data.get("images_dir") + component_names_metadata = data.get("component_names_metadata", {}) + + logger.info( + f"📨 FIJI SERVER: Received batch message with {len(items)} items" + ) if not items: - return {'status': 'success', 'message': 'Empty batch'} + return {"status": "success", "message": "Empty batch"} # CRITICAL: Copy data from shared memory IMMEDIATELY # This must happen before we send ack, so worker doesn't close shared memory copied_items = self._copy_items_from_shared_memory(items) # Queue copied items for debounced processing - self._queue_for_debounced_processing(copied_items, display_config_dict, images_dir, component_names_metadata) + self._queue_for_debounced_processing( + copied_items, + display_config_dict, + images_dir, + component_names_metadata, + ) else: # Single image message (fallback) copied_items = self._copy_items_from_shared_memory([data]) self._queue_for_debounced_processing( - copied_items, - data.get('display_config', {}), - data.get('images_dir') + copied_items, data.get("display_config", {}), data.get("images_dir") ) - return {'status': 'success', 'message': 'Data copied, queued for processing'} + return { + "status": "success", + "message": "Data copied, queued for processing", + } except Exception as e: - logger.error(f"📨 FIJI SERVER: Error processing message: {e}", exc_info=True) - return {'status': 'error', 'message': str(e)} - - def _process_items_from_batch(self, items: List[Dict[str, Any]], display_config_dict: Dict[str, Any], images_dir: str = None, component_names_metadata: Dict[str, Any] = None): + logger.error( + f"📨 FIJI SERVER: Error processing message: {e}", exc_info=True + ) + return {"status": "error", "message": str(e)} + + def _process_items_from_batch( + self, + items: List[Dict[str, Any]], + display_config_dict: Dict[str, Any], + images_dir: str = None, + component_names_metadata: Dict[str, Any] = None, + ): """ Unified processing for all item types (images, ROIs, future types). @@ -442,14 +483,14 @@ def _process_items_from_batch(self, items: List[Dict[str, Any]], display_config_ """ if not items: return - + # Default to empty dict if not provided if component_names_metadata is None: component_names_metadata = {} # STEP 1: SHARED - Get component modes and order - component_modes = display_config_dict.get('component_modes', {}) - component_order = display_config_dict['component_order'] + component_modes = display_config_dict.get("component_modes", {}) + component_order = display_config_dict["component_order"] logger.info(f"🎛️ FIJI SERVER: Component modes: {component_modes}") logger.info(f"🎛️ FIJI SERVER: Component order: {component_order}") @@ -459,33 +500,26 @@ def _process_items_from_batch(self, items: List[Dict[str, Any]], display_config_ for comp_name in component_order: unique_vals = set() for item in items: - meta = item.get('metadata', {}) + meta = item.get("metadata", {}) if comp_name in meta: unique_vals.add(meta[comp_name]) component_unique_values[comp_name] = unique_vals - logger.info(f"🔍 FIJI SERVER: Component cardinality: {[(c, len(v)) for c, v in component_unique_values.items()]}") - - # STEP 3: SHARED - Organize components by mode using component_modes directly - # For Fiji, we don't use cardinality filtering - component_modes define the mapping - # This ensures consistent CZT coordinate mapping regardless of batch size - result = { - 'window': [], - 'channel': [], - 'slice': [], - 'frame': [] - } - - for comp_name in component_order: - mode = component_modes[comp_name] - result[mode].append(comp_name) - - organized = result + logger.info( + f"🔍 FIJI SERVER: Component cardinality: {[(c, len(v)) for c, v in component_unique_values.items()]}" + ) - window_components = organized['window'] - channel_components = organized['channel'] - slice_components = organized['slice'] - frame_components = organized['frame'] + # STEP 3: SHARED - Generic projection/grouping via polystore receiver core + projection = group_items_by_component_modes( + items, + component_modes=component_modes, + component_order=component_order, + images_dir=images_dir, + ) + window_components = projection.window_components + channel_components = projection.channel_components + slice_components = projection.slice_components + frame_components = projection.frame_components logger.info(f"🗂️ FIJI SERVER: Dimension mapping:") logger.info(f" WINDOW: {window_components}") @@ -493,48 +527,34 @@ def _process_items_from_batch(self, items: List[Dict[str, Any]], display_config_ logger.info(f" SLICE: {slice_components}") logger.info(f" FRAME: {frame_components}") - # STEP 4: SHARED - Group items by window components - windows = {} - for item in items: - meta = item.get('metadata', {}) - data_type_str = item.get('data_type') - - # Build window key from window components - # Note: 'source' now represents step_name during pipeline execution, - # so images and ROIs from the same step automatically have matching source values - window_key_parts = [] - for comp in window_components: - if comp in meta: - value = meta[comp] - # Legacy ROI normalization: only needed when loading from disk (image viewer context) - # During pipeline execution, source=step_name for both images and ROIs, so they match - if comp == 'source' and images_dir and data_type_str == 'rois': - # Check if source looks like a subdirectory (not a step name) - # Step names don't contain path separators or end with '_results' - if '_results' in str(value) or '/' in str(value): - from pathlib import Path - value = Path(images_dir).name # Extract subdirectory name from full path - window_key_parts.append(f"{comp}_{value}") - window_key = "_".join(window_key_parts) if window_key_parts else "default_window" - - if window_key not in windows: - windows[window_key] = [] - windows[window_key].append(item) - - # STEP 5: Process each window group - for window_key, window_items in windows.items(): + # STEP 4: Process each window group + for window_key, window_items in projection.windows.items(): + if window_key in projection.fixed_window_labels: + self.window_fixed_labels[window_key] = projection.fixed_window_labels[ + window_key + ] self._process_window_group( - window_key, window_items, display_config_dict, - channel_components, slice_components, frame_components, - component_names_metadata + window_key, + window_items, + display_config_dict, + window_components, + channel_components, + slice_components, + frame_components, + component_names_metadata, ) - def _process_window_group(self, window_key: str, items: List[Dict[str, Any]], - display_config_dict: Dict[str, Any], - channel_components: List[str], - slice_components: List[str], - frame_components: List[str], - component_names_metadata: Dict[str, Any] = None): + def _process_window_group( + self, + window_key: str, + items: List[Dict[str, Any]], + display_config_dict: Dict[str, Any], + window_components: List[str], + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + component_names_metadata: Dict[str, Any] = None, + ): """ Process all items for a single window group. @@ -552,16 +572,28 @@ def _process_window_group(self, window_key: str, items: List[Dict[str, Any]], # Lock to prevent concurrent batches from overwriting each other's hyperstacks # (race condition in fast sequential mode where multiple debounce timers fire) with self._hyperstack_lock: - self._process_window_group_locked(window_key, items, display_config_dict, - channel_components, slice_components, frame_components, - component_names_metadata) - - def _process_window_group_locked(self, window_key: str, items: List[Dict[str, Any]], - display_config_dict: Dict[str, Any], - channel_components: List[str], - slice_components: List[str], - frame_components: List[str], - component_names_metadata: Dict[str, Any] = None): + self._process_window_group_locked( + window_key, + items, + display_config_dict, + window_components, + channel_components, + slice_components, + frame_components, + component_names_metadata, + ) + + def _process_window_group_locked( + self, + window_key: str, + items: List[Dict[str, Any]], + display_config_dict: Dict[str, Any], + window_components: List[str], + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + component_names_metadata: Dict[str, Any] = None, + ): """ Process window group with hyperstack lock held. @@ -588,47 +620,69 @@ def _process_window_group_locked(self, window_key: str, items: List[Dict[str, An stored = self.window_dimension_values[window_key] # Merge: combine stored and current, preserving order and uniqueness - channel_values = self._merge_dimension_values(stored['channel'], current_channel_values) - slice_values = self._merge_dimension_values(stored['slice'], current_slice_values) - frame_values = self._merge_dimension_values(stored['frame'], current_frame_values) + channel_values = self._merge_dimension_values( + stored["channel"], current_channel_values + ) + slice_values = self._merge_dimension_values( + stored["slice"], current_slice_values + ) + frame_values = self._merge_dimension_values( + stored["frame"], current_frame_values + ) # Log if dimensions expanded - if (len(channel_values) > len(stored['channel']) or - len(slice_values) > len(stored['slice']) or - len(frame_values) > len(stored['frame'])): - logger.info(f"🔬 FIJI SERVER: Expanded dimensions for window '{window_key}': " - f"{len(stored['channel'])}→{len(channel_values)}C, " - f"{len(stored['slice'])}→{len(slice_values)}Z, " - f"{len(stored['frame'])}→{len(frame_values)}T") + if ( + len(channel_values) > len(stored["channel"]) + or len(slice_values) > len(stored["slice"]) + or len(frame_values) > len(stored["frame"]) + ): + logger.info( + f"🔬 FIJI SERVER: Expanded dimensions for window '{window_key}': " + f"{len(stored['channel'])}→{len(channel_values)}C, " + f"{len(stored['slice'])}→{len(slice_values)}Z, " + f"{len(stored['frame'])}→{len(frame_values)}T" + ) else: - logger.info(f"🔬 FIJI SERVER: Reusing stored dimension values for window '{window_key}'") + logger.info( + f"🔬 FIJI SERVER: Reusing stored dimension values for window '{window_key}'" + ) else: # First batch for this window - use current values channel_values = current_channel_values slice_values = current_slice_values frame_values = current_frame_values - logger.info(f"🔬 FIJI SERVER: First batch for window '{window_key}': " - f"{len(channel_values)}C x {len(slice_values)}Z x {len(frame_values)}T") + logger.info( + f"🔬 FIJI SERVER: First batch for window '{window_key}': " + f"{len(channel_values)}C x {len(slice_values)}Z x {len(frame_values)}T" + ) + + # Fixed labels for window-level components (constant within this window) + first_meta = items[0].get("metadata", {}) if items else {} + self.window_fixed_labels[window_key] = [ + (comp, first_meta[comp]) for comp in window_components if comp in first_meta + ] # Store merged values for future batches self.window_dimension_values[window_key] = { - 'channel': channel_values, - 'slice': slice_values, - 'frame': frame_values + "channel": channel_values, + "slice": slice_values, + "frame": frame_values, } # STEP 2: Group items by data_type (convert string to enum) items_by_type = {} for item in items: - data_type_str = item.get('data_type') + data_type_str = item.get("data_type") # Convert string to StreamingDataType enum - if data_type_str == 'image': + if data_type_str == "image": data_type = StreamingDataType.IMAGE - elif data_type_str == 'rois': + elif data_type_str == "rois": data_type = StreamingDataType.ROIS else: - logger.warning(f"🔬 FIJI SERVER: Unknown data type string: {data_type_str}") + logger.warning( + f"🔬 FIJI SERVER: Unknown data type string: {data_type_str}" + ) continue if data_type not in items_by_type: @@ -640,19 +694,29 @@ def _process_window_group_locked(self, window_key: str, items: List[Dict[str, An handler = _FIJI_ITEM_HANDLERS.get(data_type) if handler is None: - logger.warning(f"🔬 FIJI SERVER: No handler registered for type {data_type}") + logger.warning( + f"🔬 FIJI SERVER: No handler registered for type {data_type}" + ) continue # Call handler with shared coordinate space handler( - self, window_key, type_items, display_config_dict, - channel_components, slice_components, frame_components, - channel_values, slice_values, frame_values, - component_names_metadata + self, + window_key, + type_items, + display_config_dict, + channel_components, + slice_components, + frame_components, + channel_values, + slice_values, + frame_values, + component_names_metadata, ) - def _merge_dimension_values(self, stored_values: List[tuple], - new_values: List[tuple]) -> List[tuple]: + def _merge_dimension_values( + self, stored_values: List[tuple], new_values: List[tuple] + ) -> List[tuple]: """ Merge stored and new dimension values, preserving order and uniqueness. @@ -671,14 +735,14 @@ def _merge_dimension_values(self, stored_values: List[tuple], # This ensures consistent ordering even with mixed types def sort_key(value_tuple): return tuple( - (0 if isinstance(v, (int, float)) else 1, str(v)) - for v in value_tuple + (0 if isinstance(v, (int, float)) else 1, str(v)) for v in value_tuple ) return sorted(merged, key=sort_key) - def _collect_dimension_values_from_items(self, items: List[Dict[str, Any]], - component_list: List[str]) -> List[tuple]: + def _collect_dimension_values_from_items( + self, items: List[Dict[str, Any]], component_list: List[str] + ) -> List[tuple]: """ Collect unique dimension values from items for coordinate mapping. @@ -694,7 +758,7 @@ def _collect_dimension_values_from_items(self, items: List[Dict[str, Any]], unique_values = set() for item in items: - meta = item.get('metadata', {}) + meta = item.get("metadata", {}) # Build tuple of values for this dimension (fail loud if missing) value_tuple = tuple(meta[comp] for comp in component_list) @@ -704,15 +768,17 @@ def _collect_dimension_values_from_items(self, items: List[Dict[str, Any]], # Convert each element to (type_priority, str_value) for comparison def sort_key(value_tuple): return tuple( - (0 if isinstance(v, (int, float)) else 1, str(v)) - for v in value_tuple + (0 if isinstance(v, (int, float)) else 1, str(v)) for v in value_tuple ) return sorted(unique_values, key=sort_key) - def _get_dimension_index(self, metadata: Dict[str, Any], - component_list: List[str], - dimension_values: List[tuple]) -> int: + def _get_dimension_index( + self, + metadata: Dict[str, Any], + component_list: List[str], + dimension_values: List[tuple], + ) -> int: """ Get index in dimension_values for metadata components. @@ -732,58 +798,94 @@ def _get_dimension_index(self, metadata: Dict[str, Any], try: return dimension_values.index(key) except ValueError: - logger.warning(f"🔬 FIJI SERVER: Dimension value {key} not found in {dimension_values}") + logger.warning( + f"🔬 FIJI SERVER: Dimension value {key} not found in {dimension_values}" + ) return -1 - def _add_slices_to_existing_hyperstack(self, existing_imp, new_images: List[Dict[str, Any]], - window_key: str, channel_components: List[str], - slice_components: List[str], frame_components: List[str], - display_config_dict: Dict[str, Any], - channel_values: List[tuple] = None, - slice_values: List[tuple] = None, - frame_values: List[tuple] = None): + def _add_slices_to_existing_hyperstack( + self, + existing_imp, + new_images: List[Dict[str, Any]], + window_key: str, + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + display_config_dict: Dict[str, Any], + channel_values: List[tuple] = None, + slice_values: List[tuple] = None, + frame_values: List[tuple] = None, + ): """ Incrementally add new slices to an existing hyperstack WITHOUT rebuilding. This avoids the expensive min/max recalculation that happens when rebuilding. """ - import numpy as np - from multiprocessing import shared_memory - import scyjava as sj - - # Import ImageJ classes - ShortProcessor = sj.jimport('ij.process.ShortProcessor') - # Get existing metadata existing_images = self.hyperstack_metadata[window_key] # Build lookup of existing images by coordinates + # CRITICAL: Use same key construction as new images to avoid false positives existing_lookup = {} for img_data in existing_images: - meta = img_data['metadata'] - c_key = tuple(meta[comp] for comp in channel_components) if channel_components else () - z_key = tuple(meta[comp] for comp in slice_components) if slice_components else () - t_key = tuple(meta[comp] for comp in frame_components) if frame_components else () + meta = img_data["metadata"] + c_key = ( + tuple(meta.get(comp) for comp in channel_components) + if channel_components + else () + ) + z_key = ( + tuple(meta.get(comp) for comp in slice_components) + if slice_components + else () + ) + t_key = ( + tuple(meta.get(comp) for comp in frame_components) + if frame_components + else () + ) existing_lookup[(c_key, z_key, t_key)] = img_data # Get existing stack and dimensions stack = existing_imp.getStack() + stack_width = stack.getWidth() + stack_height = stack.getHeight() old_nChannels = existing_imp.getNChannels() old_nSlices = existing_imp.getNSlices() old_nFrames = existing_imp.getNFrames() # Collect dimension values from existing images - existing_channel_values = self.collect_dimension_values(existing_images, channel_components) - existing_slice_values = self.collect_dimension_values(existing_images, slice_components) - existing_frame_values = self.collect_dimension_values(existing_images, frame_components) + existing_channel_values = self.collect_dimension_values( + existing_images, channel_components + ) + existing_slice_values = self.collect_dimension_values( + existing_images, slice_components + ) + existing_frame_values = self.collect_dimension_values( + existing_images, frame_components + ) # Process new images and check if dimensions changed + # CRITICAL: Check existing hyperstack dimensions, not existing images + # Use channel_values/slice_values/frame_values from hyperstack metadata to detect actual dimension changes new_coords_added = [] for img_data in new_images: - meta = img_data['metadata'] - c_key = tuple(meta[comp] for comp in channel_components) if channel_components else () - z_key = tuple(meta[comp] for comp in slice_components) if slice_components else () - t_key = tuple(meta[comp] for comp in frame_components) if frame_components else () + meta = img_data["metadata"] + c_key = ( + tuple(meta.get(comp) for comp in channel_components) + if channel_components + else () + ) + z_key = ( + tuple(meta.get(comp) for comp in slice_components) + if slice_components + else () + ) + t_key = ( + tuple(meta.get(comp) for comp in frame_components) + if frame_components + else () + ) coord = (c_key, z_key, t_key) @@ -794,46 +896,157 @@ def _add_slices_to_existing_hyperstack(self, existing_imp, new_images: List[Dict # Update lookup (new images override existing at same coordinates) existing_lookup[coord] = img_data - # ImageJ hyperstacks have fixed dimensions - we need to rebuild when adding new slices - # But we can preserve display ranges to avoid expensive min/max recalculation - all_images = list(existing_lookup.values()) + # Check if dimensions actually changed + # NEW LOGIC: Check if any coordinate has a dimension value not in existing dimension values + # This is more accurate than checking if coordinates exist in existing_lookup + dimensions_changed = False + spatial_dimensions_changed = False + for coord in new_coords_added: + c_key, z_key, t_key = coord + if c_key and c_key not in existing_channel_values: + dimensions_changed = True + break + if z_key and z_key not in existing_slice_values: + dimensions_changed = True + break + if t_key and t_key not in existing_frame_values: + dimensions_changed = True + break + + if not dimensions_changed: + for img_data in new_images: + np_data = self._extract_2d_plane(img_data["data"]) + height, width = np_data.shape[-2:] + if height > stack_height or width > stack_width: + spatial_dimensions_changed = True + break + + if not dimensions_changed and not spatial_dimensions_changed: + # OPTIMIZATION: Only slice replacements - do INCREMENTAL UPDATE + # Replace only changed slices in existing ImageStack WITHOUT rebuilding + # This avoids recalculating contrast for ALL images + logger.info( + f"🔬 FIJI SERVER: ⚡ INCREMENTAL: Replacing {len(new_images)} slices in '{window_key}' (no rebuild, no recalc)" + ) - logger.info(f"🔬 FIJI SERVER: 🔄 REBUILDING: Merging {len(new_images)} new images into '{window_key}' (total: {len(all_images)} images, existing had {len(existing_images)})") + # Map of new pixel data to replace + new_pixel_data = {} + for img_data in new_images: + np_data = self._extract_2d_plane(img_data["data"]) + np_data = self._pad_plane_to_shape(np_data, stack_height, stack_width) + + meta = img_data["metadata"] + c_key = ( + tuple(meta.get(comp) for comp in channel_components) + if channel_components + else () + ) + z_key = ( + tuple(meta.get(comp) for comp in slice_components) + if slice_components + else () + ) + t_key = ( + tuple(meta.get(comp) for comp in frame_components) + if frame_components + else () + ) - # Store display range before rebuilding - display_ranges = [] - if old_nChannels > 0: - for c in range(1, old_nChannels + 1): - try: - existing_imp.setC(c) - display_ranges.append((existing_imp.getDisplayRangeMin(), existing_imp.getDisplayRangeMax())) - except Exception as e: - logger.warning(f"Failed to get display range for channel {c}: {e}") - # Use default range if we can't get it - display_ranges.append((0, 255)) + coord = (c_key, z_key, t_key) - # Close old hyperstack - existing_imp.close() + # Calculate slice index in ImageJ CZT order + c_idx = channel_values.index(c_key) if c_key else 0 + z_idx = slice_values.index(z_key) if z_key else 0 + t_idx = frame_values.index(t_key) if t_key else 0 - # Build new hyperstack with all images (old + new) - # Pass is_new=False and preserved_display_ranges to avoid recalculation - # Pass dimension values to use shared coordinate space - self._build_new_hyperstack( - all_images, window_key, channel_components, slice_components, - frame_components, display_config_dict, is_new=False, - preserved_display_ranges=display_ranges, - channel_values=channel_values, slice_values=slice_values, frame_values=frame_values - ) + nChannels = len(channel_values) if channel_values else 1 + nSlices = len(slice_values) if slice_values else 1 + nFrames = len(frame_values) if frame_values else 1 + + slice_idx = (t_idx * nSlices * nChannels) + (z_idx * nChannels) + c_idx + + new_pixel_data[slice_idx] = np_data + + # CRITICAL: Replace ALL changed slices in ONE call to avoid repeated min/max recalc + # This is the key to avoiding fibonacci performance + for slice_idx, np_data in new_pixel_data.items(): + stack.setPixels(slice_idx, np_data) + + # Update metadata + self.hyperstack_metadata[window_key] = list(existing_lookup.values()) + + # CRITICAL: Do NOT apply auto-contrast during incremental updates! + # Only repaint window - auto-contrast will be applied on FINAL batch + # This avoids O(n) auto-contrast for every single slice + imp = existing_imp + imp.updateAndRepaintWindow() - def _build_single_hyperstack(self, window_key: str, images: List[Dict[str, Any]], - display_config_dict: Dict[str, Any], - channel_components: List[str], - slice_components: List[str], - frame_components: List[str], - channel_values: List[tuple] = None, - slice_values: List[tuple] = None, - frame_values: List[tuple] = None, - component_names_metadata: Dict[str, Any] = None): + logger.info( + f"🔬 FIJI SERVER: ✅ Incremental update complete for '{window_key}'" + ) + else: + # Dimensions changed - need FULL REBUILD + # ImageJ hyperstacks have fixed dimensions (C/Z/T and spatial width/height) + # Preserve display ranges to avoid expensive min/max recalculation + all_images = list(existing_lookup.values()) + logger.info( + f"🔬 FIJI SERVER: 🔄 REBUILDING: Merging {len(new_images)} new images into " + f"'{window_key}' (total: {len(all_images)} images, existing had {len(existing_images)}, " + f"spatial_changed={spatial_dimensions_changed}, coord_changed={dimensions_changed})" + ) + + # Store display range before rebuilding + display_ranges = [] + if old_nChannels > 0: + for c in range(1, old_nChannels + 1): + try: + existing_imp.setC(c) + display_ranges.append( + ( + existing_imp.getDisplayRangeMin(), + existing_imp.getDisplayRangeMax(), + ) + ) + except Exception as e: + logger.warning( + f"Failed to get display range for channel {c}: {e}" + ) + # Use default range if we can't get it + display_ranges.append((0, 255)) + + # Close old hyperstack + existing_imp.close() + + # Build new hyperstack with all images (old + new) + # Pass is_new=False and preserved_display_ranges to avoid recalculation + # Pass dimension values to use shared coordinate space + self._build_new_hyperstack( + all_images, + window_key, + channel_components, + slice_components, + frame_components, + display_config_dict, + is_new=False, + preserved_display_ranges=display_ranges, + channel_values=channel_values, + slice_values=slice_values, + frame_values=frame_values, + ) + + def _build_single_hyperstack( + self, + window_key: str, + images: List[Dict[str, Any]], + display_config_dict: Dict[str, Any], + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + channel_values: List[tuple] = None, + slice_values: List[tuple] = None, + frame_values: List[tuple] = None, + component_names_metadata: Dict[str, Any] = None, + ): """ Build or update a single ImageJ hyperstack from images. @@ -854,36 +1067,73 @@ def _build_single_hyperstack(self, window_key: str, images: List[Dict[str, Any]] import scyjava as sj # Import ImageJ classes using scyjava - ImageStack = sj.jimport('ij.ImageStack') - ImagePlus = sj.jimport('ij.ImagePlus') - CompositeImage = sj.jimport('ij.CompositeImage') - ShortProcessor = sj.jimport('ij.process.ShortProcessor') + ImageStack = sj.jimport("ij.ImageStack") + ImagePlus = sj.jimport("ij.ImagePlus") + CompositeImage = sj.jimport("ij.CompositeImage") + ShortProcessor = sj.jimport("ij.process.ShortProcessor") # Check if we have an existing hyperstack to merge into existing_imp = self.hyperstacks.get(window_key) + + # Check if the existing hyperstack is still valid (window not closed by user) + is_valid = False + if existing_imp is not None: + try: + # Check if the ImagePlus still has a valid window + window = existing_imp.getWindow() + is_valid = window is not None and window.isVisible() + except Exception: + is_valid = False + + # If existing hyperstack is closed/invalid, treat as new hyperstack + if existing_imp is not None and not is_valid: + logger.info( + f"🔬 FIJI SERVER: Existing hyperstack '{window_key}' is closed/invalid, creating new one" + ) + existing_imp = None + is_new_hyperstack = existing_imp is None if not is_new_hyperstack: # INCREMENTAL UPDATE: Add only new slices to existing hyperstack - logger.info(f"🔬 FIJI SERVER: ⚡ BATCH UPDATE: Adding {len(images)} new images to existing hyperstack '{window_key}'") + logger.info( + f"🔬 FIJI SERVER: ⚡ BATCH UPDATE: Adding {len(images)} new images to existing hyperstack '{window_key}'" + ) self._add_slices_to_existing_hyperstack( - existing_imp, images, window_key, - channel_components, slice_components, frame_components, + existing_imp, + images, + window_key, + channel_components, + slice_components, + frame_components, display_config_dict, - channel_values=channel_values, slice_values=slice_values, frame_values=frame_values + channel_values=channel_values, + slice_values=slice_values, + frame_values=frame_values, ) return # NEW HYPERSTACK: Build from scratch - logger.info(f"🔬 FIJI SERVER: ✨ NEW HYPERSTACK: Creating '{window_key}' with {len(images)} images") + logger.info( + f"🔬 FIJI SERVER: ✨ NEW HYPERSTACK: Creating '{window_key}' with {len(images)} images" + ) self._build_new_hyperstack( - images, window_key, channel_components, slice_components, - frame_components, display_config_dict, is_new=True, - channel_values=channel_values, slice_values=slice_values, frame_values=frame_values, - component_names_metadata=component_names_metadata + images, + window_key, + channel_components, + slice_components, + frame_components, + display_config_dict, + is_new=True, + channel_values=channel_values, + slice_values=slice_values, + frame_values=frame_values, + component_names_metadata=component_names_metadata, ) - def _build_image_lookup(self, images, channel_components, slice_components, frame_components): + def _build_image_lookup( + self, images, channel_components, slice_components, frame_components + ): """Build coordinate lookup dict from images. Returns: @@ -891,23 +1141,85 @@ def _build_image_lookup(self, images, channel_components, slice_components, fram """ image_lookup = {} for img_data in images: - meta = img_data['metadata'] - c_key = tuple(meta[comp] for comp in channel_components) if channel_components else () - z_key = tuple(meta[comp] for comp in slice_components) if slice_components else () - t_key = tuple(meta[comp] for comp in frame_components) if frame_components else () - image_lookup[(c_key, z_key, t_key)] = img_data['data'] + meta = img_data["metadata"] + c_key = ( + tuple(meta[comp] for comp in channel_components) + if channel_components + else () + ) + z_key = ( + tuple(meta[comp] for comp in slice_components) + if slice_components + else () + ) + t_key = ( + tuple(meta[comp] for comp in frame_components) + if frame_components + else () + ) + image_lookup[(c_key, z_key, t_key)] = img_data["data"] return image_lookup - def _create_imagestack_from_images(self, image_lookup, channel_values, slice_values, frame_values, - width, height, channel_components, slice_components, frame_components): + def _extract_2d_plane(self, np_data): + """Extract a 2D plane using the canonical Fiji rule for 3D payloads.""" + if np_data.ndim == 3: + return np_data[np_data.shape[0] // 2] + return np_data + + def _pad_plane_to_shape(self, np_data, target_height: int, target_width: int): + """Pad a 2D plane to target spatial shape using zero fill.""" + import numpy as np + + current_height, current_width = np_data.shape[-2:] + if current_height == target_height and current_width == target_width: + return np_data + + if current_height > target_height or current_width > target_width: + raise ValueError( + f"Cannot pad plane {(current_height, current_width)} " + f"to smaller target {(target_height, target_width)}" + ) + + padded = np.zeros((target_height, target_width), dtype=np_data.dtype) + padded[:current_height, :current_width] = np_data + return padded + + def _compute_target_spatial_shape(self, all_images: List[Dict[str, Any]]) -> tuple: + """Compute target (height, width) as max spatial shape across all images.""" + max_height = 0 + max_width = 0 + for img_data in all_images: + plane = self._extract_2d_plane(img_data["data"]) + height, width = plane.shape[-2:] + max_height = max(max_height, height) + max_width = max(max_width, width) + return max_height, max_width + + def _create_imagestack_from_images( + self, + image_lookup, + channel_values, + slice_values, + frame_values, + width, + height, + channel_components, + slice_components, + frame_components, + ): """Create ImageJ ImageStack from image lookup dict. Returns: ImageJ ImageStack object """ import scyjava as sj - ImageStack = sj.jimport('ij.ImageStack') - ShortProcessor = sj.jimport('ij.process.ShortProcessor') + import numpy as np + import jpype + + ImageStack = sj.jimport("ij.ImageStack") + ShortProcessor = sj.jimport("ij.process.ShortProcessor") + ByteProcessor = sj.jimport("ij.process.ByteProcessor") + FloatProcessor = sj.jimport("ij.process.FloatProcessor") stack = ImageStack(width, height) @@ -918,26 +1230,61 @@ def _create_imagestack_from_images(self, image_lookup, channel_values, slice_val key = (c_key, z_key, t_key) if key in image_lookup: - np_data = image_lookup[key] - - # Handle 3D data (take middle slice) - if np_data.ndim == 3: - np_data = np_data[np_data.shape[0] // 2] - - # Convert to ImageProcessor - temp_imp = self.ij.py.to_imageplus(np_data) - processor = temp_imp.getProcessor() + np_data = self._extract_2d_plane(image_lookup[key]) + np_data = self._pad_plane_to_shape(np_data, height, width) + + # CRITICAL: Create processor directly using JPype array conversion + # Avoid to_imageplus() which triggers min/max calculation for EACH slice + np_data = np.ascontiguousarray(np_data).flatten() + + # Convert numpy array to Java array using JPype buffer protocol (fast, no copy) + # This bypasses scyjava's type checking and works with all numpy dtypes + if np_data.dtype == np.uint8: + # ByteProcessor expects signed bytes in Java + java_array = jpype.JArray(jpype.JByte)( + np_data.astype(np.int8) + ) + processor = ByteProcessor(width, height, java_array, None) + elif np_data.dtype in (np.uint16, np.int16): + # ShortProcessor expects signed shorts in Java + java_array = jpype.JArray(jpype.JShort)( + np_data.astype(np.int16) + ) + processor = ShortProcessor(width, height, java_array, None) + elif np_data.dtype in (np.float32, np.float64): + java_array = jpype.JArray(jpype.JFloat)( + np_data.astype(np.float32) + ) + processor = FloatProcessor(width, height, java_array) + else: + # Default to ShortProcessor + java_array = jpype.JArray(jpype.JShort)( + np_data.astype(np.int16) + ) + processor = ShortProcessor(width, height, java_array, None) # Build label label_parts = [] if channel_components: - c_str = "_".join(str(v) for v in c_key) if isinstance(c_key, tuple) else str(c_key) + c_str = ( + "_".join(str(v) for v in c_key) + if isinstance(c_key, tuple) + else str(c_key) + ) label_parts.append(f"C{c_str}") if slice_components: - z_str = "_".join(str(v) for v in z_key) if isinstance(z_key, tuple) else str(z_key) + z_str = ( + "_".join(str(v) for v in z_key) + if isinstance(z_key, tuple) + else str(z_key) + ) label_parts.append(f"Z{z_str}") if frame_components: - t_str = "_".join(str(v) for v in t_key) if isinstance(t_key, tuple) else str(t_key) + t_str = ( + "_".join(str(v) for v in t_key) + if isinstance(t_key, tuple) + else str(t_key) + ) label_parts.append(f"T{t_str}") label = "_".join(label_parts) if label_parts else "slice" @@ -962,13 +1309,15 @@ def _convert_to_hyperstack(self, imp, nChannels, nSlices, nFrames, window_key): # Convert to HyperStack to enable proper Z/T slider behavior if nSlices > 1 or nFrames > 1 or nChannels > 1: - HyperStackConverter = sj.jimport('ij.plugin.HyperStackConverter') - imp = HyperStackConverter.toHyperStack(imp, nChannels, nSlices, nFrames, "xyczt", "Composite") + HyperStackConverter = sj.jimport("ij.plugin.HyperStackConverter") + imp = HyperStackConverter.toHyperStack( + imp, nChannels, nSlices, nFrames, "xyczt", "Composite" + ) imp.setTitle(window_key) # Convert to CompositeImage if multiple channels if nChannels > 1: - CompositeImage = sj.jimport('ij.CompositeImage') + CompositeImage = sj.jimport("ij.CompositeImage") if not isinstance(imp, CompositeImage): comp = CompositeImage(imp, CompositeImage.COMPOSITE) comp.setTitle(window_key) @@ -976,15 +1325,27 @@ def _convert_to_hyperstack(self, imp, nChannels, nSlices, nFrames, window_key): return imp - def _apply_display_settings(self, imp, lut_name, auto_contrast, nChannels, preserved_ranges=None): + def _apply_display_settings( + self, + imp, + window_key: str, + lut_name, + auto_contrast, + nChannels, + preserved_ranges=None, + skip_auto_contrast=False, + ): """Apply LUT and display settings to ImagePlus. Args: imp: ImagePlus to modify + window_key: Hyperstack window key (for logging) lut_name: LUT name to apply auto_contrast: Whether to apply auto-contrast nChannels: Number of channels preserved_ranges: Optional list of (min, max) tuples per channel + skip_auto_contrast: If True, skip auto-contrast even if auto_contrast=True + Used during loading to avoid O(n) recalc on every slice """ if preserved_ranges: # Restore preserved display ranges @@ -994,32 +1355,51 @@ def _apply_display_settings(self, imp, lut_name, auto_contrast, nChannels, prese imp.setDisplayRange(min_val, max_val) else: # Apply LUT and auto-contrast for new hyperstacks - if lut_name not in ['Grays', 'grays'] and nChannels == 1: + if lut_name not in ["Grays", "grays"] and nChannels == 1: try: self.ij.IJ.run(imp, lut_name, "") except Exception as e: - logger.warning(f"🔬 FIJI SERVER: Failed to apply LUT {lut_name}: {e}") + logger.warning( + f"🔬 FIJI SERVER: Failed to apply LUT {lut_name}: {e}" + ) - if auto_contrast: + # CRITICAL: Skip auto-contrast during incremental updates to avoid O(n) recalc on every slice + # Only apply on final batch when skip_auto_contrast=False + if auto_contrast and not skip_auto_contrast: try: self.ij.IJ.run(imp, "Enhance Contrast", "saturated=0.35") + logger.info( + f"🔬 FIJI SERVER: ✅ Applied auto-contrast to '{window_key}' " + f"(nChannels={nChannels})" + ) except Exception as e: - logger.warning(f"🔬 FIJI SERVER: Failed to apply auto-contrast: {e}") - - def _create_dimension_label_overlay(self, imp, channel_components: List[str], slice_components: List[str], - frame_components: List[str], channel_values: List[tuple], - slice_values: List[tuple], frame_values: List[tuple], - component_names_metadata: Dict[str, Any]): + logger.warning( + f"🔬 FIJI SERVER: Failed to apply auto-contrast: {e}" + ) + + def _create_dimension_label_overlay( + self, + window_key: str, + imp, + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + channel_values: List[tuple], + slice_values: List[tuple], + frame_values: List[tuple], + component_names_metadata: Dict[str, Any], + ): """ Create a text overlay showing current dimension labels (like napari's text_overlay). - + This creates an actual on-screen text overlay that updates when dimensions change, matching napari's behavior. - + Args: + window_key: Window key for fixed window-component labels imp: ImagePlus instance channel_components: Components mapped to Channel dimension - slice_components: Components mapped to Slice dimension + slice_components: Components mapped to Slice dimension frame_components: Components mapped to Frame dimension channel_values: Channel dimension values slice_values: Slice dimension values @@ -1027,17 +1407,17 @@ def _create_dimension_label_overlay(self, imp, channel_components: List[str], sl component_names_metadata: Dict mapping component names to {id: name} dicts """ import scyjava as sj - + try: logger.info(f"🏷️ FIJI SERVER: Creating dimension label overlay") - + # Component abbreviation mapping COMPONENT_ABBREV = { "channel": "Ch", "z_index": "Z", "timepoint": "T", "site": "Site", - "well": "Well" + "well": "Well", } # Build label text showing actual component names and values @@ -1048,7 +1428,7 @@ def add_component_label(comp_name, comp_value): if comp_name in component_names_metadata: metadata_dict = component_names_metadata[comp_name] metadata_name = metadata_dict.get(str(comp_value)) - if metadata_name and str(metadata_name).lower() != 'none': + if metadata_name and str(metadata_name).lower() != "none": # Show metadata name (e.g., "Brightfield", "DAPI") label_parts.append(metadata_name) else: @@ -1059,7 +1439,11 @@ def add_component_label(comp_name, comp_value): # No metadata - use abbreviated component name + index abbrev = COMPONENT_ABBREV.get(comp_name, comp_name) label_parts.append(f"{abbrev} {comp_value}") - + + # Window-level fixed labels (e.g., Well when mapped to window mode) + for comp_name, comp_value in self.window_fixed_labels.get(window_key, []): + add_component_label(comp_name, comp_value) + # Channel dimension - show actual channel component values if channel_components and channel_values: current_channel = imp.getChannel() # 1-indexed @@ -1068,7 +1452,7 @@ def add_component_label(comp_name, comp_value): for comp_idx, comp_name in enumerate(channel_components): comp_value = channel_tuple[comp_idx] add_component_label(comp_name, comp_value) - + # Slice dimension - show actual slice component values if slice_components and slice_values: current_slice = imp.getSlice() # 1-indexed @@ -1077,7 +1461,7 @@ def add_component_label(comp_name, comp_value): for comp_idx, comp_name in enumerate(slice_components): comp_value = slice_tuple[comp_idx] add_component_label(comp_name, comp_value) - + # Frame dimension - show actual frame component values (well, site, timepoint, etc.) if frame_components and frame_values: current_frame = imp.getFrame() # 1-indexed @@ -1086,70 +1470,93 @@ def add_component_label(comp_name, comp_value): for comp_idx, comp_name in enumerate(frame_components): comp_value = frame_tuple[comp_idx] add_component_label(comp_name, comp_value) - + if not label_parts: - logger.info(f"🏷️ FIJI SERVER: No dimensions to label (no channels/slices/frames)") + logger.info( + f"🏷️ FIJI SERVER: No dimensions to label (no channels/slices/frames)" + ) # Still create a test overlay to verify the mechanism works label_parts = ["TEST OVERLAY"] - + label_text = " | ".join(label_parts) logger.info(f"🏷️ FIJI SERVER: Creating overlay with text: '{label_text}'") - + # Create text overlay using ImageJ Overlay API - TextRoi = sj.jimport('ij.gui.TextRoi') - Overlay = sj.jimport('ij.gui.Overlay') - Font = sj.jimport('java.awt.Font') - Color = sj.jimport('java.awt.Color') - + TextRoi = sj.jimport("ij.gui.TextRoi") + Overlay = sj.jimport("ij.gui.Overlay") + Font = sj.jimport("java.awt.Font") + Color = sj.jimport("java.awt.Color") + # Position text in top-left corner x = 10 y = 20 - + # Create text ROI with white fill color text_roi = TextRoi(x, y, label_text) text_roi.setFont(Font("SansSerif", Font.BOLD, 16)) - + # Set fill color to white (this is what shows) text_roi.setFillColor(Color.WHITE) # Set stroke (outline) color to black for contrast text_roi.setStrokeColor(Color.BLACK) text_roi.setStrokeWidth(2.0) - + # Create or get overlay overlay = imp.getOverlay() if overlay is None: overlay = Overlay() - + # Clear any existing overlays first overlay.clear() - + # Add text ROI to overlay overlay.add(text_roi) imp.setOverlay(overlay) - - logger.info(f"🏷️ FIJI SERVER: Text overlay created successfully with white text") - + + logger.info( + f"🏷️ FIJI SERVER: Text overlay created successfully with white text" + ) + # Add listener to update overlay when hyperstack position changes - self._add_dimension_change_listener(imp, channel_components, slice_components, frame_components, - channel_values, slice_values, frame_values, - component_names_metadata) - + self._add_dimension_change_listener( + window_key, + imp, + channel_components, + slice_components, + frame_components, + channel_values, + slice_values, + frame_values, + component_names_metadata, + ) + except Exception as e: - logger.error(f"🏷️ FIJI SERVER: Failed to create dimension label overlay: {e}", exc_info=True) - - def _add_dimension_change_listener(self, imp, channel_components: List[str], slice_components: List[str], - frame_components: List[str], channel_values: List[tuple], - slice_values: List[tuple], frame_values: List[tuple], - component_names_metadata: Dict[str, Any]): + logger.error( + f"🏷️ FIJI SERVER: Failed to create dimension label overlay: {e}", + exc_info=True, + ) + + def _add_dimension_change_listener( + self, + window_key: str, + imp, + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + channel_values: List[tuple], + slice_values: List[tuple], + frame_values: List[tuple], + component_names_metadata: Dict[str, Any], + ): """ Add a listener to update the text overlay when hyperstack position changes. - + Uses AdjustmentListener on the StackWindow scrollbars to detect dimension changes. Based on ImageJ source: ij.gui.StackWindow implements AdjustmentListener for its scrollbars. """ import scyjava as sj import jpype - + try: # Component abbreviation mapping COMPONENT_ABBREV = { @@ -1157,7 +1564,7 @@ def _add_dimension_change_listener(self, imp, channel_components: List[str], sli "z_index": "Z", "timepoint": "T", "site": "Site", - "well": "Well" + "well": "Well", } # Helper function to build label text from current position @@ -1169,7 +1576,7 @@ def add_component_label(comp_name, comp_value): if comp_name in component_names_metadata: metadata_dict = component_names_metadata[comp_name] metadata_name = metadata_dict.get(str(comp_value)) - if metadata_name and str(metadata_name).lower() != 'none': + if metadata_name and str(metadata_name).lower() != "none": # Use metadata name (e.g., "DAPI") label_parts.append(metadata_name) else: @@ -1181,6 +1588,12 @@ def add_component_label(comp_name, comp_value): abbrev = COMPONENT_ABBREV.get(comp_name, comp_name) label_parts.append(f"{abbrev} {comp_value}") + # Window-level fixed labels (e.g., Well when mapped to window mode) + for comp_name, comp_value in self.window_fixed_labels.get( + window_key, [] + ): + add_component_label(comp_name, comp_value) + # Channel if channel_components and channel_values: current_channel = imp.getChannel() @@ -1206,17 +1619,19 @@ def add_component_label(comp_name, comp_value): add_component_label(comp_name, frame_tuple[comp_idx]) return " | ".join(label_parts) if label_parts else "TEST OVERLAY" - + # Helper function to update overlay text def update_overlay(): try: label_text = build_label_text() - logger.info(f"🔄 FIJI SERVER: Updating overlay text to: '{label_text}'") + logger.info( + f"🔄 FIJI SERVER: Updating overlay text to: '{label_text}'" + ) - TextRoi = sj.jimport('ij.gui.TextRoi') - Overlay = sj.jimport('ij.gui.Overlay') - Font = sj.jimport('java.awt.Font') - Color = sj.jimport('java.awt.Color') + TextRoi = sj.jimport("ij.gui.TextRoi") + Overlay = sj.jimport("ij.gui.Overlay") + Font = sj.jimport("java.awt.Font") + Color = sj.jimport("java.awt.Color") text_roi = TextRoi(10, 20, label_text) text_roi.setFont(Font("SansSerif", Font.BOLD, 16)) @@ -1233,20 +1648,22 @@ def update_overlay(): canvas.repaint() logger.info(f"🔄 FIJI SERVER: Overlay updated successfully") except Exception as e: - logger.error(f"🔄 FIJI SERVER: Error updating overlay: {e}", exc_info=True) - + logger.error( + f"🔄 FIJI SERVER: Error updating overlay: {e}", exc_info=True + ) + # Get the StackWindow and add AdjustmentListener to its scrollbars window = imp.getWindow() if window is not None: try: # Define listener class using jpype @JImplements decorator - @jpype.JImplements('java.awt.event.AdjustmentListener') + @jpype.JImplements("java.awt.event.AdjustmentListener") class DimensionScrollbarListener: @jpype.JOverride def adjustmentValueChanged(self, event): # Called when user scrolls through C/Z/T dimensions update_overlay() - + listener = DimensionScrollbarListener() # StackWindow has cSelector, zSelector, tSelector scrollbars @@ -1255,14 +1672,19 @@ def adjustmentValueChanged(self, event): added = [] logger.info(f"🏷️ FIJI SERVER: Window type: {type(window).__name__}") - logger.info(f"🏷️ FIJI SERVER: Hyperstack: {imp.getNChannels()}C x {imp.getNSlices()}Z x {imp.getNFrames()}T") + logger.info( + f"🏷️ FIJI SERVER: Hyperstack: {imp.getNChannels()}C x {imp.getNSlices()}Z x {imp.getNFrames()}T" + ) # Scrollbars are AWT components, not fields # Find all ScrollbarWithLabel components and attach listeners import scyjava as sj + try: components = window.getComponents() - logger.info(f"🏷️ FIJI SERVER: Window has {len(components)} components") + logger.info( + f"🏷️ FIJI SERVER: Window has {len(components)} components" + ) scrollbar_count = 0 for i, comp in enumerate(components): @@ -1270,49 +1692,139 @@ def adjustmentValueChanged(self, event): logger.info(f"🏷️ FIJI SERVER: Component {i}: {comp_type}") # ScrollbarWithLabel is the hyperstack dimension scrollbar - if 'ScrollbarWithLabel' in comp_type: + if "ScrollbarWithLabel" in comp_type: try: # Just attach listener to all scrollbars # The update_overlay() function will read current position from imp comp.addAdjustmentListener(listener) scrollbar_count += 1 - added.append(f'scrollbar_{i}') - logger.info(f"🏷️ FIJI SERVER: Added listener to scrollbar {i}") + added.append(f"scrollbar_{i}") + logger.info( + f"🏷️ FIJI SERVER: Added listener to scrollbar {i}" + ) except Exception as e: - logger.warning(f"🏷️ FIJI SERVER: Could not attach listener: {e}") + logger.warning( + f"🏷️ FIJI SERVER: Could not attach listener: {e}" + ) - logger.info(f"🏷️ FIJI SERVER: Attached listeners to {scrollbar_count} scrollbars") + logger.info( + f"🏷️ FIJI SERVER: Attached listeners to {scrollbar_count} scrollbars" + ) except Exception as e: - logger.warning(f"🏷️ FIJI SERVER: Could not enumerate components: {e}") + logger.warning( + f"🏷️ FIJI SERVER: Could not enumerate components: {e}" + ) if added: - logger.info(f"🏷️ FIJI SERVER: Added scrollbar listeners for: {', '.join(added)}") + logger.info( + f"🏷️ FIJI SERVER: Added scrollbar listeners for: {', '.join(added)}" + ) else: - logger.warning(f"🏷️ FIJI SERVER: No scrollbars found to attach listener (not a hyperstack?)") - + logger.warning( + f"🏷️ FIJI SERVER: No scrollbars found to attach listener (not a hyperstack?)" + ) + + # Add WindowListener to detect when user closes the window + # Capture closure variables explicitly + captured_window_key = window_key + captured_hyperstacks = self.hyperstacks + captured_metadata = self.hyperstack_metadata + captured_dim_values = self.window_dimension_values + captured_fixed_labels = self.window_fixed_labels + captured_lock = self._hyperstack_lock + + @jpype.JImplements("java.awt.event.WindowListener") + class WindowCloseListener: + @jpype.JOverride + def windowClosing(self, event): + with captured_lock: + if captured_window_key in captured_hyperstacks: + closed_imp = captured_hyperstacks.pop( + captured_window_key, None + ) + if closed_imp is not None: + try: + closed_imp.close() + except Exception: + pass + if captured_window_key in captured_metadata: + captured_metadata.pop(captured_window_key, None) + if captured_window_key in captured_dim_values: + captured_dim_values.pop( + captured_window_key, None + ) + if captured_window_key in captured_fixed_labels: + captured_fixed_labels.pop( + captured_window_key, None + ) + logger.info( + f"🔬 FIJI SERVER: Cleaned up hyperstack '{captured_window_key}' after window close" + ) + + @jpype.JOverride + def windowClosed(self, event): + pass + + @jpype.JOverride + def windowOpened(self, event): + pass + + @jpype.JOverride + def windowIconified(self, event): + pass + + @jpype.JOverride + def windowDeiconified(self, event): + pass + + @jpype.JOverride + def windowActivated(self, event): + pass + + @jpype.JOverride + def windowDeactivated(self, event): + pass + + window.addWindowListener(WindowCloseListener()) + logger.info( + f"🏷️ FIJI SERVER: Added window close listener for '{window_key}'" + ) + except Exception as e: - logger.warning(f"Could not add scrollbar listeners: {e}", exc_info=True) + logger.warning( + f"Could not add scrollbar listeners: {e}", exc_info=True + ) else: logger.warning(f"🏷️ FIJI SERVER: No window found, cannot add listeners") - + except Exception as e: - logger.error(f"🏷️ FIJI SERVER: Failed to add dimension change listener: {e}", exc_info=True) - - def _set_dimension_labels(self, imp, channel_components: List[str], slice_components: List[str], - frame_components: List[str], channel_values: List[tuple], - slice_values: List[tuple], frame_values: List[tuple], - component_names_metadata: Dict[str, Any]): + logger.error( + f"🏷️ FIJI SERVER: Failed to add dimension change listener: {e}", + exc_info=True, + ) + + def _set_dimension_labels( + self, + imp, + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + channel_values: List[tuple], + slice_values: List[tuple], + frame_values: List[tuple], + component_names_metadata: Dict[str, Any], + ): """ Set dimension labels on ImagePlus using component metadata. - + Sets both: 1. ImageJ property metadata (for channel selector UI) 2. Text overlay (for on-screen display like napari) - + Args: imp: ImagePlus instance channel_components: Components mapped to Channel dimension - slice_components: Components mapped to Slice dimension + slice_components: Components mapped to Slice dimension frame_components: Components mapped to Frame dimension channel_values: Channel dimension values slice_values: Slice dimension values @@ -1320,13 +1832,19 @@ def _set_dimension_labels(self, imp, channel_components: List[str], slice_compon component_names_metadata: Dict mapping component names to {id: name} dicts """ try: - logger.info(f"🏷️ FIJI SERVER: _set_dimension_labels called with {len(channel_values) if channel_values else 0} channels") + logger.info( + f"🏷️ FIJI SERVER: _set_dimension_labels called with {len(channel_values) if channel_values else 0} channels" + ) logger.info(f"🏷️ FIJI SERVER: channel_components = {channel_components}") - logger.info(f"🏷️ FIJI SERVER: component_names_metadata = {component_names_metadata}") - + logger.info( + f"🏷️ FIJI SERVER: component_names_metadata = {component_names_metadata}" + ) + # Set channel labels if channel_components and channel_values: - logger.info(f"🏷️ FIJI SERVER: Setting labels for {len(channel_values)} channels") + logger.info( + f"🏷️ FIJI SERVER: Setting labels for {len(channel_values)} channels" + ) for idx, channel_tuple in enumerate(channel_values, start=1): # Build label from component metadata label_parts = [] @@ -1336,8 +1854,10 @@ def _set_dimension_labels(self, imp, channel_components: List[str], slice_compon if comp_name in component_names_metadata: metadata_dict = component_names_metadata[comp_name] metadata_name = metadata_dict.get(comp_value) - logger.debug(f"🏷️ Found metadata for {comp_name}[{comp_value}]: {metadata_name}") - if metadata_name and str(metadata_name).lower() != 'none': + logger.debug( + f"🏷️ Found metadata for {comp_name}[{comp_value}]: {metadata_name}" + ) + if metadata_name and str(metadata_name).lower() != "none": # Use metadata name (e.g., "DAPI") label_parts.append(metadata_name) else: @@ -1368,32 +1888,45 @@ def _set_dimension_labels(self, imp, channel_components: List[str], slice_compon # Set the label property (ImageJ uses "Label1", "Label2", etc. for channels) label = " | ".join(label_parts) if label_parts else f"Ch{idx}" imp.setProperty(f"Label{idx}", label) - logger.info(f"🏷️ FIJI SERVER: Set channel {idx} label: '{label}' (property key: 'Label{idx}')") - + logger.info( + f"🏷️ FIJI SERVER: Set channel {idx} label: '{label}' (property key: 'Label{idx}')" + ) + # Verify the property was set verified_label = imp.getProperty(f"Label{idx}") - logger.info(f"🏷️ FIJI SERVER: Verified channel {idx} label: '{verified_label}'") - + logger.info( + f"🏷️ FIJI SERVER: Verified channel {idx} label: '{verified_label}'" + ) + # Note: ImageJ doesn't have built-in properties for slice/frame labels like it does for channels # Those would require custom overlays or other visualization methods - + except Exception as e: logger.warning(f"Failed to set dimension labels: {e}", exc_info=True) - def _build_new_hyperstack(self, all_images: List[Dict[str, Any]], window_key: str, - channel_components: List[str], slice_components: List[str], - frame_components: List[str], display_config_dict: Dict[str, Any], - is_new: bool, preserved_display_ranges: List[tuple] = None, - channel_values: List[tuple] = None, - slice_values: List[tuple] = None, - frame_values: List[tuple] = None, - component_names_metadata: Dict[str, Any] = None): + def _build_new_hyperstack( + self, + all_images: List[Dict[str, Any]], + window_key: str, + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + display_config_dict: Dict[str, Any], + is_new: bool, + preserved_display_ranges: List[tuple] = None, + channel_values: List[tuple] = None, + slice_values: List[tuple] = None, + frame_values: List[tuple] = None, + component_names_metadata: Dict[str, Any] = None, + ): """Build a new hyperstack from scratch.""" import scyjava as sj # Collect dimension values (use provided values if available, otherwise compute) if channel_values is None: - channel_values = self.collect_dimension_values(all_images, channel_components) + channel_values = self.collect_dimension_values( + all_images, channel_components + ) if slice_values is None: slice_values = self.collect_dimension_values(all_images, slice_components) if frame_values is None: @@ -1403,76 +1936,113 @@ def _build_new_hyperstack(self, all_images: List[Dict[str, Any]], window_key: st nSlices = len(slice_values) nFrames = len(frame_values) - logger.info(f"🔬 FIJI SERVER: Building hyperstack '{window_key}': {nChannels}C x {nSlices}Z x {nFrames}T") + logger.info( + f"🔬 FIJI SERVER: Building hyperstack '{window_key}': {nChannels}C x {nSlices}Z x {nFrames}T" + ) if not all_images: logger.error(f"🔬 FIJI SERVER: No images provided for '{window_key}'") return - # Get spatial dimensions - first_img = all_images[0]['data'] - height, width = first_img.shape[-2:] + # Get target spatial dimensions (mixed shapes allowed by zero-padding smaller planes) + height, width = self._compute_target_spatial_shape(all_images) + logger.info( + f"🔬 FIJI SERVER: Target stack shape for '{window_key}' is {height}x{width}" + ) # Build image lookup - image_lookup = self._build_image_lookup(all_images, channel_components, slice_components, frame_components) + image_lookup = self._build_image_lookup( + all_images, channel_components, slice_components, frame_components + ) + + # CRITICAL: Calculate global min/max ONCE from all images BEFORE creating stack + # This avoids O(n) calculation per slice during stack creation + import numpy as np + + logger.info( + f"🔬 FIJI SERVER: Calculating global min/max from {len(all_images)} images" + ) + global_min = float("inf") + global_max = float("-inf") + for img_data in all_images: + np_data = self._extract_2d_plane(img_data["data"]) + img_min = float(np.min(np_data)) + img_max = float(np.max(np_data)) + global_min = min(global_min, img_min) + global_max = max(global_max, img_max) + logger.info(f"🔬 FIJI SERVER: Global min/max: {global_min} - {global_max}") # Create ImageStack stack = self._create_imagestack_from_images( - image_lookup, channel_values, slice_values, frame_values, - width, height, channel_components, slice_components, frame_components + image_lookup, + channel_values, + slice_values, + frame_values, + width, + height, + channel_components, + slice_components, + frame_components, ) # Create ImagePlus - ImagePlus = sj.jimport('ij.ImagePlus') + ImagePlus = sj.jimport("ij.ImagePlus") imp = ImagePlus(window_key, stack) + # Set display range using pre-calculated global min/max + # This prevents ImageJ from scanning all pixels again + imp.setDisplayRange(global_min, global_max) + # Convert to hyperstack imp = self._convert_to_hyperstack(imp, nChannels, nSlices, nFrames, window_key) # Set dimension labels from metadata (e.g., channel names like "DAPI", "GFP") - logger.info(f"🏷️ FIJI SERVER: component_names_metadata = {component_names_metadata}") - + logger.info( + f"🏷️ FIJI SERVER: component_names_metadata = {component_names_metadata}" + ) + # Note: Text overlay will be created AFTER imp.show() so the window exists for listeners - + title_suffix = "" if component_names_metadata: logger.info(f"🏷️ FIJI SERVER: Setting dimension labels for {window_key}") # Set property metadata for channel selector UI - self._set_dimension_labels(imp, channel_components, slice_components, frame_components, - channel_values, slice_values, frame_values, - component_names_metadata) - + self._set_dimension_labels( + imp, + channel_components, + slice_components, + frame_components, + channel_values, + slice_values, + frame_values, + component_names_metadata, + ) + # For single-channel images, add channel name to window title since no slider appears if nChannels == 1 and channel_components and channel_values: first_comp = channel_components[0] # channel_values is a list of tuples, e.g., [(1,)] first_value_tuple = channel_values[0] first_value_str = str(first_value_tuple[0]) - + if first_comp in component_names_metadata: comp_metadata = component_names_metadata[first_comp] if first_value_str in comp_metadata: channel_name = comp_metadata[first_value_str] - if channel_name and str(channel_name).lower() != 'none': + if channel_name and str(channel_name).lower() != "none": title_suffix = f" [{channel_name}]" - logger.info(f"🏷️ FIJI SERVER: Adding channel name to window title: {title_suffix}") + logger.info( + f"🏷️ FIJI SERVER: Adding channel name to window title: {title_suffix}" + ) else: - logger.info(f"🏷️ FIJI SERVER: No component_names_metadata available for {window_key}") - + logger.info( + f"🏷️ FIJI SERVER: No component_names_metadata available for {window_key}" + ) + # Update window title with suffix if present if title_suffix: imp.setTitle(f"{window_key}{title_suffix}") - # Apply display settings - lut_name = display_config_dict.get('lut', 'Grays') - auto_contrast = display_config_dict.get('auto_contrast', True) - # Only apply auto-contrast for NEW hyperstacks; for updates, use preserved ranges - # This avoids expensive O(n²) recalculation when adding slices incrementally - self._apply_display_settings( - imp, lut_name, auto_contrast if is_new else False, nChannels, - preserved_ranges=None if is_new else preserved_display_ranges - ) - # Close old hyperstack if rebuilding if window_key in self.hyperstacks: self.hyperstacks[window_key].close() @@ -1485,18 +2055,44 @@ def _build_new_hyperstack(self, all_images: List[Dict[str, Any]], window_key: st # Show after storing (imp.show() may be async on Swing thread) imp.show() - logger.info(f"🔬 FIJI SERVER: Displayed hyperstack '{window_key}' with {stack.getSize()} slices") + # Apply display settings after the window is shown. + # This keeps auto-contrast off the per-slice update path. + lut_name = display_config_dict.get("lut", "Grays") + auto_contrast = display_config_dict.get("auto_contrast", True) + self._apply_display_settings( + imp, + window_key, + lut_name, + auto_contrast if is_new else False, + nChannels, + preserved_ranges=None if is_new else preserved_display_ranges, + skip_auto_contrast=(not is_new) and (preserved_display_ranges is not None), + ) + + logger.info( + f"🔬 FIJI SERVER: Displayed hyperstack '{window_key}' with {stack.getSize()} slices" + ) # NOW create text overlay AFTER window exists (so listeners can be attached) - logger.info(f"🏷️ FIJI SERVER: Creating dimension label overlay for {window_key}") - self._create_dimension_label_overlay(imp, channel_components, slice_components, frame_components, - channel_values, slice_values, frame_values, - component_names_metadata or {}) + logger.info( + f"🏷️ FIJI SERVER: Creating dimension label overlay for {window_key}" + ) + self._create_dimension_label_overlay( + window_key, + imp, + channel_components, + slice_components, + frame_components, + channel_values, + slice_values, + frame_values, + component_names_metadata or {}, + ) # Send acknowledgments for img_data in all_images: - if image_id := img_data.get('image_id'): - self._send_ack(image_id, status='success') + if image_id := img_data.get("image_id"): + self._send_ack(image_id, status="success") def request_shutdown(self): """Request graceful shutdown.""" @@ -1505,15 +2101,19 @@ def request_shutdown(self): @register_fiji_handler(StreamingDataType.IMAGE) -def _handle_images_for_window(self, window_key: str, items: List[Dict[str, Any]], - display_config_dict: Dict[str, Any], - channel_components: List[str], - slice_components: List[str], - frame_components: List[str], - channel_values: List[tuple], - slice_values: List[tuple], - frame_values: List[tuple], - component_names_metadata: Dict[str, Any] = None): +def _handle_images_for_window( + self, + window_key: str, + items: List[Dict[str, Any]], + display_config_dict: Dict[str, Any], + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + channel_values: List[tuple], + slice_values: List[tuple], + frame_values: List[tuple], + component_names_metadata: Dict[str, Any] = None, +): """ Handle images for a window group. @@ -1523,16 +2123,20 @@ def _handle_images_for_window(self, window_key: str, items: List[Dict[str, Any]] # Convert to standard format expected by hyperstack builder image_data_list = [] for item in items: - if 'data' in item: + if "data" in item: # Already copied from shared memory (debounced path) - image_data_list.append({ - 'data': item['data'], - 'metadata': item.get('metadata', {}), - 'image_id': item.get('image_id') - }) - elif 'shm_name' in item: + image_data_list.append( + { + "data": item["data"], + "metadata": item.get("metadata", {}), + "image_id": item.get("image_id"), + } + ) + elif "shm_name" in item: # Need to load from shared memory (direct path - shouldn't happen with debouncing) - loaded = self.load_images_from_shared_memory([item], error_callback=self._send_ack) + loaded = self.load_images_from_shared_memory( + [item], error_callback=self._send_ack + ) image_data_list.extend(loaded) if not image_data_list: @@ -1541,23 +2145,33 @@ def _handle_images_for_window(self, window_key: str, items: List[Dict[str, Any]] # Delegate to existing hyperstack building logic # Pass dimension values so it uses shared coordinate space self._build_single_hyperstack( - window_key, image_data_list, display_config_dict, - channel_components, slice_components, frame_components, - channel_values, slice_values, frame_values, - component_names_metadata + window_key, + image_data_list, + display_config_dict, + channel_components, + slice_components, + frame_components, + channel_values, + slice_values, + frame_values, + component_names_metadata, ) @register_fiji_handler(StreamingDataType.ROIS) -def _handle_rois_for_window(self, window_key: str, items: List[Dict[str, Any]], - display_config_dict: Dict[str, Any], - channel_components: List[str], - slice_components: List[str], - frame_components: List[str], - channel_values: List[tuple], - slice_values: List[tuple], - frame_values: List[tuple], - component_names_metadata: Dict[str, Any] = None): +def _handle_rois_for_window( + self, + window_key: str, + items: List[Dict[str, Any]], + display_config_dict: Dict[str, Any], + channel_components: List[str], + slice_components: List[str], + frame_components: List[str], + channel_values: List[tuple], + slice_values: List[tuple], + frame_values: List[tuple], + component_names_metadata: Dict[str, Any] = None, +): """ Handle ROIs for a window group. @@ -1568,7 +2182,7 @@ def _handle_rois_for_window(self, window_key: str, items: List[Dict[str, Any]], import scyjava as sj # Get or create RoiManager - MUST be done on EDT to avoid Swing threading issues - RoiManager = sj.jimport('ij.plugin.frame.RoiManager') + RoiManager = sj.jimport("ij.plugin.frame.RoiManager") # Try to get existing instance first (doesn't require EDT) rm = RoiManager.getInstance() @@ -1580,10 +2194,12 @@ def _handle_rois_for_window(self, window_key: str, items: List[Dict[str, Any]], from jpype import JImplements, JOverride except ImportError: # Fallback: create RoiManager directly (may cause EDT issues with IPC mode) - logger.warning("JPype not available, creating RoiManager without EDT safety (may fail with IPC mode)") + logger.warning( + "JPype not available, creating RoiManager without EDT safety (may fail with IPC mode)" + ) rm = RoiManager() else: - SwingUtilities = sj.jimport('javax.swing.SwingUtilities') + SwingUtilities = sj.jimport("javax.swing.SwingUtilities") # Use a holder to get the RoiManager out of the Runnable rm_holder = [None] @@ -1591,7 +2207,7 @@ def _handle_rois_for_window(self, window_key: str, items: List[Dict[str, Any]], # Create a Java Runnable using JPype's JImplements decorator # Note: Decorators are evaluated at class definition time, but since this is # inside an if block that only executes when rm is None, it's safe - @JImplements('java.lang.Runnable') + @JImplements("java.lang.Runnable") class CreateRoiManagerRunnable: @JOverride def run(self): @@ -1611,19 +2227,25 @@ def run(self): total_rois_added = 0 for roi_item in items: - rois_encoded = roi_item.get('rois', []) + rois_encoded = roi_item.get("rois", []) if not rois_encoded: - if image_id := roi_item.get('image_id'): - self._send_ack(image_id, status='success') + if image_id := roi_item.get("image_id"): + self._send_ack(image_id, status="success") continue - meta = roi_item.get('metadata', {}) - file_path = roi_item.get('path', 'unknown') + meta = roi_item.get("metadata", {}) + file_path = roi_item.get("path", "unknown") logger.info(f"🔬 FIJI SERVER: ROI metadata: {meta}") - logger.info(f"🔬 FIJI SERVER: Channel components: {channel_components}, values: {channel_values}") - logger.info(f"🔬 FIJI SERVER: Slice components: {slice_components}, values: {slice_values}") - logger.info(f"🔬 FIJI SERVER: Frame components: {frame_components}, values: {frame_values}") + logger.info( + f"🔬 FIJI SERVER: Channel components: {channel_components}, values: {channel_values}" + ) + logger.info( + f"🔬 FIJI SERVER: Slice components: {slice_components}, values: {slice_values}" + ) + logger.info( + f"🔬 FIJI SERVER: Frame components: {frame_components}, values: {frame_values}" + ) # Map metadata to CZT indices using SHARED coordinate space c_index = self._get_dimension_index(meta, channel_components, channel_values) @@ -1635,13 +2257,16 @@ def run(self): z_value = z_index + 1 if z_index >= 0 else 1 t_value = t_index + 1 if t_index >= 0 else 1 - logger.info(f"🔬 FIJI SERVER: ROI '{file_path}' position: C={c_value}, Z={z_value}, T={t_value} (from indices {c_index}, {z_index}, {t_index})") + logger.info( + f"🔬 FIJI SERVER: ROI '{file_path}' position: C={c_value}, Z={z_value}, T={t_value} (from indices {c_index}, {z_index}, {t_index})" + ) # Decode and add ROIs roi_bytes_list = FijiROIConverter.decode_rois_from_transmission(rois_encoded) # Extract base name from path for ROI naming from pathlib import Path + base_name = Path(file_path).stem # Get filename without extension for roi_idx, roi_bytes in enumerate(roi_bytes_list): @@ -1660,13 +2285,15 @@ def run(self): rm.addRoi(java_roi) total_rois_added += 1 - if image_id := roi_item.get('image_id'): - self._send_ack(image_id, status='success') + if image_id := roi_item.get("image_id"): + self._send_ack(image_id, status="success") if not rm.isVisible(): rm.setVisible(True) - logger.info(f"🔬 FIJI SERVER: Added {total_rois_added} ROIs to group {group_id} ('{window_key}') with shared coordinate space") + logger.info( + f"🔬 FIJI SERVER: Added {total_rois_added} ROIs to group {group_id} ('{window_key}') with shared coordinate space" + ) # Make handlers instance methods by binding them to the class @@ -1674,7 +2301,14 @@ def run(self): FijiViewerServer._handle_rois_for_window = _handle_rois_for_window -def _fiji_viewer_server_process(port: int, viewer_title: str, display_config, log_file_path: str = None, transport_mode: OpenHCSTransportMode = OpenHCSTransportMode.IPC): +def _fiji_viewer_server_process( + port: int, + viewer_title: str, + display_config, + log_file_path: str = None, + transport_mode: OpenHCSTransportMode = OpenHCSTransportMode.IPC, + zmq_config=None, +): """ Fiji viewer server process function. @@ -1686,17 +2320,31 @@ def _fiji_viewer_server_process(port: int, viewer_title: str, display_config, lo display_config: FijiDisplayConfig instance log_file_path: Path to log file (for client discovery via ping/pong) transport_mode: ZMQ transport mode (IPC or TCP) + zmq_config: ZMQ configuration object (optional, uses default if None) """ try: import zmq # Create ZMQ server instance (inherits from ZMQServer ABC) - server = FijiViewerServer(port, viewer_title, display_config, log_file_path, transport_mode) - + if zmq_config is None: + from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG + + zmq_config = OPENHCS_ZMQ_CONFIG + server = FijiViewerServer( + port, + viewer_title, + display_config, + log_file_path, + transport_mode, + zmq_config, + ) + # Start the server (binds sockets, initializes PyImageJ) server.start() - - logger.info(f"🔬 FIJI SERVER: Server started on port {port}, control port {port + 1000}") + + logger.info( + f"🔬 FIJI SERVER: Server started on port {port}, control port {port + 1000}" + ) logger.info("🔬 FIJI SERVER: Waiting for images...") # Message processing loop @@ -1727,25 +2375,34 @@ def _fiji_viewer_server_process(port: int, viewer_title: str, display_config, lo ack_response = server.process_image_message(message) except Exception as e: # ANY error during processing - send error response to maintain socket state - logger.error(f"🔬 FIJI SERVER: Error processing image message: {e}", exc_info=True) - ack_response = {'status': 'error', 'message': str(e)} + logger.error( + f"🔬 FIJI SERVER: Error processing image message: {e}", + exc_info=True, + ) + ack_response = {"status": "error", "message": str(e)} # Step 3: ALWAYS send response (even if it's an error response) try: server.data_socket.send_json(ack_response) - logger.info(f"🔬 FIJI SERVER: Sent ack to worker: {ack_response['status']}") + logger.info( + f"🔬 FIJI SERVER: Sent ack to worker: {ack_response['status']}" + ) except Exception as e: # If send fails, the socket is likely broken - log and continue - logger.error(f"🔬 FIJI SERVER: Failed to send ack on data socket: {e}", exc_info=True) + logger.error( + f"🔬 FIJI SERVER: Failed to send ack on data socket: {e}", + exc_info=True, + ) time.sleep(0.001) # 1ms sleep - faster polling for multiprocessing - + logger.info("🔬 FIJI SERVER: Shutting down...") server.stop() - + except Exception as e: logger.error(f"🔬 FIJI SERVER: Error: {e}") import traceback + traceback.print_exc() finally: logger.info("🔬 FIJI SERVER: Process terminated") diff --git a/openhcs/runtime/napari_stream_visualizer.py b/openhcs/runtime/napari_stream_visualizer.py index b0237db81..c9d89c24f 100644 --- a/openhcs/runtime/napari_stream_visualizer.py +++ b/openhcs/runtime/napari_stream_visualizer.py @@ -25,9 +25,11 @@ from qtpy.QtCore import QTimer from polystore.filemanager import FileManager -from polystore.backend_registry import register_cleanup_callback from openhcs.utils.import_utils import optional_import -from openhcs.core.config import TransportMode as OpenHCSTransportMode, NapariStreamingConfig +from openhcs.core.config import ( + TransportMode as OpenHCSTransportMode, + NapariStreamingConfig, +) from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG from zmqruntime.config import TransportMode as ZMQTransportMode from zmqruntime.streaming import StreamingVisualizerServer, VisualizerProcessManager @@ -86,9 +88,6 @@ def _cleanup_global_viewer() -> None: _global_viewer_process = None -register_cleanup_callback(_cleanup_global_viewer) - - def _parse_component_info_from_path(path_str: str): """ Fallback component parsing from path (used when component metadata unavailable). @@ -135,7 +134,7 @@ def _build_nd_shapes(layer_items, stack_components): Returns: Tuple of (all_shapes_nd, all_shape_types, all_properties) """ - from polystore.roi_converters import NapariROIConverter + from openhcs.runtime.roi_converters import NapariROIConverter all_shapes_nd = [] all_shape_types = [] @@ -198,13 +197,15 @@ def _build_nd_points(layer_items, stack_components, component_values=None): if component_values is None: component_values = {} for comp in stack_components: - values = sorted(set(item["components"].get(comp, 0) for item in layer_items)) + values = sorted( + set(item["components"].get(comp, 0) for item in layer_items) + ) component_values[comp] = values for item in layer_items: points_data = item["data"] # List of shape dicts from ROI converter components = item["components"] - + # DEBUG: Log what we actually have logger.info(f"🐛 DEBUG: points_data type: {type(points_data)}") if isinstance(points_data, list) and len(points_data) > 0: @@ -223,21 +224,23 @@ def _build_nd_points(layer_items, stack_components, component_values=None): # Only process 'points' type entries if shape_dict.get("type") != "points": continue - + coordinates = shape_dict.get("coordinates", []) metadata = shape_dict.get("metadata", {}) - + # coordinates is a list of [y, x] pairs # Prepend stack dimensions to each point: [y, x] -> [stack_idx, ..., y, x] for coord in coordinates: nd_coord = prepend_dims + list(coord) all_points_nd.append(nd_coord) - + # Track properties for this point all_properties["label"].append(metadata.get("label", "")) all_properties["component"].append(metadata.get("component", 0)) - return np.array(all_points_nd) if all_points_nd else np.empty((0, 2 + len(stack_components))), all_properties + return np.array(all_points_nd) if all_points_nd else np.empty( + (0, 2 + len(stack_components)) + ), all_properties def _build_nd_image_array(layer_items, stack_components, component_values=None): @@ -270,7 +273,7 @@ def _build_nd_image_array(layer_items, stack_components, component_values=None): # Single item, single component, no global values - just return as-is # (Will be wrapped in extra dimension if needed by caller) return layer_items[0]["data"] - + # Multiple stack components OR using global component values - create multi-dimensional array if component_values is None: # Derive from layer items (old behavior when no global tracker) @@ -315,7 +318,7 @@ def _create_or_update_layer( ): """ Create or update a Napari layer of any type. - + All layers are handled identically: if layer exists, remove and recreate. This ensures consistent behavior across all layer types. @@ -349,17 +352,21 @@ def _create_or_update_layer( add_method = getattr(viewer, f"add_{layer_type}") new_layer = add_method(data, name=layer_name, **layer_kwargs) layers[layer_name] = new_layer - + # Log with appropriate count/info if layer_type == "shapes": - count = len(data) if hasattr(data, '__len__') else 0 - logger.info(f"🔬 NAPARI PROCESS: Created {layer_type} layer {layer_name} with {count} shapes") + count = len(data) if hasattr(data, "__len__") else 0 + logger.info( + f"🔬 NAPARI PROCESS: Created {layer_type} layer {layer_name} with {count} shapes" + ) elif layer_type == "points": - count = len(data) if hasattr(data, '__len__') else 0 - logger.info(f"🔬 NAPARI PROCESS: Created {layer_type} layer {layer_name} with {count} points") + count = len(data) if hasattr(data, "__len__") else 0 + logger.info( + f"🔬 NAPARI PROCESS: Created {layer_type} layer {layer_name} with {count} points" + ) else: logger.info(f"🔬 NAPARI PROCESS: Created {layer_type} layer {layer_name}") - + return new_layer @@ -396,9 +403,7 @@ def _create_or_update_shapes_layer( ) -def _create_or_update_points_layer( - viewer, layers, layer_name, points_data, properties -): +def _create_or_update_points_layer(viewer, layers, layer_name, points_data, properties): """Create or update a Napari points layer.""" return _create_or_update_layer( viewer, @@ -413,7 +418,11 @@ def _create_or_update_points_layer( # Populate registry now that helper functions are defined -from openhcs.constants.streaming import StreamingDataType +from polystore.streaming_constants import StreamingDataType +from polystore.streaming.receivers.napari import ( + normalize_component_layout, + build_layer_key, +) _DATA_TYPE_HANDLERS = { StreamingDataType.IMAGE: { @@ -464,55 +473,19 @@ def _handle_component_aware_display( raise ValueError(f"No component metadata available for path: {path}") component_info = component_metadata - # Build component_modes and component_order from config (dict or object) - component_modes = None - component_order = None - - if isinstance(display_config, dict): - cm = display_config.get("component_modes") or display_config.get( - "componentModes" - ) - if isinstance(cm, dict) and cm: - component_modes = cm - component_order = display_config["component_order"] - else: - # Handle object-like config (NapariDisplayConfig) - component_order = display_config.COMPONENT_ORDER - component_modes = {} - for component in component_order: - mode_field = f"{component}_mode" - if hasattr(display_config, mode_field): - mode_value = getattr(display_config, mode_field) - component_modes[component] = getattr( - mode_value, "value", str(mode_value) - ) - - # Generic layer naming - iterate over components in canonical order - # Components in SLICE mode create separate layers - # Components in STACK mode are combined into the same layer - - layer_key_parts = [] - for component in component_order: - mode = component_modes.get(component) - if mode == "slice" and component in component_info: - value = component_info[component] - layer_key_parts.append(f"{component}_{value}") - - layer_key = "_".join(layer_key_parts) if layer_key_parts else "default_layer" + component_modes, component_order = normalize_component_layout(display_config) + layer_key = build_layer_key( + component_info=component_info, + component_modes=component_modes, + component_order=component_order, + data_type=data_type, + ) # Log component modes for debugging logger.info( f"🔍 NAPARI PROCESS: component_modes={component_modes}, layer_key='{layer_key}'" ) - # Add "_shapes" suffix for shapes layers to distinguish from image layers - # Add "_points" suffix for points layers to distinguish from image layers - # MUST happen BEFORE reconciliation so we check the correct layer name - if data_type == StreamingDataType.SHAPES: - layer_key = f"{layer_key}_shapes" - elif data_type == StreamingDataType.POINTS: - layer_key = f"{layer_key}_points" - # Log layer key and component info for debugging logger.info( f"🔍 NAPARI PROCESS: layer_key='{layer_key}', component_info={component_info}" @@ -652,7 +625,7 @@ def __init__( viewer_type="napari", host="*", log_file_path=log_file_path, - data_socket_type=zmq.SUB, + data_socket_type=zmq.REP, transport_mode=coerce_transport_mode(transport_mode), config=OPENHCS_ZMQ_CONFIG, ) @@ -664,7 +637,7 @@ def __init__( self.component_groups = {} self.dimension_labels = {} # Store dimension label mappings: layer_key -> {component: [labels]} self.component_metadata = {} # Store component metadata from microscope handler: {component: {id: name}} - + # Global component value tracker for shared dimension mapping # Maps tuple of stack_components -> {component: set of values} # All layers with the same stack_components share the same global mapping @@ -686,63 +659,63 @@ def _setup_ack_socket(self): def _update_global_component_values(self, stack_components, layer_items): """ Update the global component value tracker with values from new items. - + All layers sharing the same stack_components will use the same global mapping, ensuring consistent component-to-index mapping across image and ROI layers. - + Args: stack_components: Tuple/list of component names (e.g., ['channel', 'well']) layer_items: List of items with 'components' dict """ # Use tuple as dict key (lists aren't hashable) components_key = tuple(stack_components) - + # Initialize if needed if components_key not in self.global_component_values: self.global_component_values[components_key] = { comp: set() for comp in stack_components } - + # Add values from these items global_values = self.global_component_values[components_key] for item in layer_items: for comp in stack_components: value = item["components"].get(comp, 0) global_values[comp].add(value) - + logger.info( f"🔬 NAPARI PROCESS: Updated global component values for {stack_components}" ) for comp, values in global_values.items(): sorted_values = sorted(values) logger.info(f"🔬 NAPARI PROCESS: {comp}: {sorted_values}") - + def _get_global_component_values(self, stack_components): """ Get the global component values for a given set of stack components. - + For indexed components (channel, z_index, timepoint), expands to include all values from 1 to max. For example, if only channel 2 is seen, returns [1, 2]. This ensures proper stack dimensions even when some indices aren't present. - + Returns a dict of {component: sorted list of values} for all components that have been seen across all layers sharing these stack components. """ components_key = tuple(stack_components) - + if components_key not in self.global_component_values: return {comp: [] for comp in stack_components} - + # Convert sets to sorted lists and expand indexed components global_values = self.global_component_values[components_key] result = {} - + # Components that should be expanded from 1 to max (1-indexed) - INDEXED_COMPONENTS = {'channel', 'z_index', 'timepoint'} - + INDEXED_COMPONENTS = {"channel", "z_index", "timepoint"} + for comp, values in global_values.items(): sorted_values = sorted(values) - + if comp in INDEXED_COMPONENTS and sorted_values: # Expand to include all indices from 1 to max # E.g., if we have [2, 4], expand to [1, 2, 3, 4] @@ -760,7 +733,7 @@ def _get_global_component_values(self, stack_components): else: # Non-indexed component (well, site, etc.) - use actual values result[comp] = sorted_values - + return result def _schedule_layer_update( @@ -833,7 +806,9 @@ def _execute_layer_update( if mode == "stack" and comp in component_info ] - logger.info(f"🔬 NAPARI PROCESS: Using stack components: {stack_components}") + logger.info( + f"🔬 NAPARI PROCESS: Using stack components: {stack_components}" + ) # Build and update the layer based on data type try: @@ -856,29 +831,29 @@ def _execute_layer_update( except Exception as e: logger.error( f"🔬 NAPARI PROCESS: Failed to update layer {layer_key}: {e}", - exc_info=True + exc_info=True, ) # Continue running - don't crash the viewer def _setup_dimension_label_handler(self, layer_key, stack_components): """ Set up event handler to update text overlay when dimensions change. - + This connects the viewer's dimension slider changes to text overlay updates, displaying categorical labels (like well IDs) instead of numeric indices. - + Args: layer_key: The layer to monitor for dimension changes stack_components: List of components that are stacked (e.g., ['well', 'channel']) """ if not self.viewer or not stack_components: return - + # Get dimension label mappings for this layer layer_labels = self.dimension_labels.get(layer_key, {}) if not layer_labels: return - + def update_dimension_label(event=None): """Update text overlay with current dimension labels.""" try: @@ -895,7 +870,7 @@ def update_dimension_label(event=None): if 0 <= idx < len(labels): label = labels[idx] # Don't show if label is None or "None" - if label and str(label).lower() != 'none': + if label and str(label).lower() != "none": label_parts.append(label) if label_parts: @@ -905,27 +880,31 @@ def update_dimension_label(event=None): except Exception as e: logger.debug(f"🔬 NAPARI PROCESS: Error updating dimension label: {e}") - + # Connect to dimension change events try: self.viewer.dims.events.current_step.connect(update_dimension_label) # Initial update update_dimension_label() - logger.info(f"🔬 NAPARI PROCESS: Set up dimension label handler for {layer_key}") + logger.info( + f"🔬 NAPARI PROCESS: Set up dimension label handler for {layer_key}" + ) except Exception as e: - logger.warning(f"🔬 NAPARI PROCESS: Failed to setup dimension label handler: {e}") + logger.warning( + f"🔬 NAPARI PROCESS: Failed to setup dimension label handler: {e}" + ) def _update_image_layer( self, layer_key, layer_items, stack_components, component_modes ): """Update an image layer with the current items.""" - + # Update global component tracker with values from these items self._update_global_component_values(stack_components, layer_items) - + # Get global component values (union of all layers with same stack_components) global_component_values = self._get_global_component_values(stack_components) - + # Check if images have different shapes and pad if needed shapes = [item["data"].shape for item in layer_items] if len(set(shapes)) > 1: @@ -966,7 +945,9 @@ def _update_image_layer( logger.info( f"🔬 NAPARI PROCESS: Building nD data for {layer_key} from {len(layer_items)} items" ) - stacked_data = _build_nd_image_array(layer_items, stack_components, global_component_values) + stacked_data = _build_nd_image_array( + layer_items, stack_components, global_component_values + ) # Determine colormap colormap = None @@ -999,7 +980,7 @@ def _update_image_layer( "z_index": "Z", "timepoint": "T", "site": "Site", - "well": "Well" + "well": "Well", } for comp in stack_components: @@ -1016,12 +997,14 @@ def _update_image_layer( # First try to get name from metadata (e.g., channel name) metadata_name = comp_metadata.get(str(v)) - if metadata_name and str(metadata_name).lower() != 'none': + if metadata_name and str(metadata_name).lower() != "none": # Use metadata name with index for clarity if comp == "channel": labels.append(f"Ch{v}: {metadata_name}") elif comp == "well": - labels.append(f"{metadata_name}") # Well names are already good (e.g., "A01") + labels.append( + f"{metadata_name}" + ) # Well names are already good (e.g., "A01") else: labels.append(f"{comp.title()} {v}: {metadata_name}") else: @@ -1030,15 +1013,15 @@ def _update_image_layer( labels.append(f"{abbrev} {v}") dimension_labels[comp] = labels - + # Store dimension labels for this layer self.dimension_labels[layer_key] = dimension_labels - + # Create or update the layer _create_or_update_image_layer( self.viewer, self.layers, layer_key, stacked_data, colormap, axis_labels ) - + # Set up dimension label handler (connects dimension changes to text overlay) self._setup_dimension_label_handler(layer_key, stack_components) @@ -1052,13 +1035,15 @@ def _update_shapes_layer( # Update global component tracker with values from these items self._update_global_component_values(stack_components, layer_items) - + # Get global component values (union of all layers with same stack_components) global_component_values = self._get_global_component_values(stack_components) # Convert shapes to label masks (much faster than individual shapes) # This happens synchronously but is fast because we're just creating arrays - labels_data = self._shapes_to_labels(layer_items, stack_components, global_component_values) + labels_data = self._shapes_to_labels( + layer_items, stack_components, global_component_values + ) # Remove existing layer if it exists if layer_key in self.layers: @@ -1085,41 +1070,44 @@ def _update_points_layer( """Update a points layer (for skeleton tracings and point-based ROIs).""" # Filter to only POINTS items (exclude IMAGE items that may share the same layer_key) points_items = [ - item for item in layer_items + item + for item in layer_items if item.get("data_type") == StreamingDataType.POINTS ] - + if not points_items: logger.warning( f"🔬 NAPARI PROCESS: No POINTS items found for {layer_key}, skipping" ) return - + logger.info( f"🔬 NAPARI PROCESS: Building points layer for {layer_key} from {len(points_items)} items (filtered from {len(layer_items)} total)" ) # Update global component tracker with ALL items (images + points) to stay in sync self._update_global_component_values(stack_components, layer_items) - + # Get global component values (union of all layers with same stack_components) global_component_values = self._get_global_component_values(stack_components) # Build nD points data using ONLY the points items BUT with global component values - points_data, properties = _build_nd_points(points_items, stack_components, global_component_values) + points_data, properties = _build_nd_points( + points_items, stack_components, global_component_values + ) # Create or update the points layer _create_or_update_points_layer( self.viewer, self.layers, layer_key, points_data, properties ) - + logger.info( f"🔬 NAPARI PROCESS: Created points layer {layer_key} with {len(points_data)} points" ) def _shapes_to_labels(self, layer_items, stack_components, component_values): """Convert shapes data to label masks. - + Args: layer_items: List of shape items to convert stack_components: List of component names for stack dimensions @@ -1140,7 +1128,9 @@ def _shapes_to_labels(self, layer_items, stack_components, component_values): first_shapes = layer_items[0]["data"] if not first_shapes: # No shapes, return empty array with reasonable default size - logger.warning("🔬 NAPARI PROCESS: No shapes data, creating default 512x512 array") + logger.warning( + "🔬 NAPARI PROCESS: No shapes data, creating default 512x512 array" + ) return np.zeros((1, 1, 512, 512), dtype=np.uint16) # Estimate image size from shape coordinates @@ -1162,7 +1152,7 @@ def _shapes_to_labels(self, layer_items, stack_components, component_values): if len(coords) > 0: max_y = max(max_y, int(np.max(coords[:, 0])) + 1) max_x = max(max_x, int(np.max(coords[:, 1])) + 1) - + # Ensure minimum valid dimensions (avoid 0x0 shapes) if max_y == 0 or max_x == 0: logger.warning( @@ -1217,8 +1207,12 @@ def _shapes_to_labels(self, layer_items, stack_components, component_values): cc = coords[:, 1].astype(int) # Clip to image bounds - valid = (rr >= 0) & (rr < labels_array.shape[-2]) & \ - (cc >= 0) & (cc < labels_array.shape[-1]) + valid = ( + (rr >= 0) + & (rr < labels_array.shape[-2]) + & (cc >= 0) + & (cc < labels_array.shape[-1]) + ) rr, cc = rr[valid], cc[valid] # Set label at the correct nD position @@ -1227,7 +1221,7 @@ def _shapes_to_labels(self, layer_items, stack_components, component_values): label_id += 1 logger.info( - f"🔬 NAPARI PROCESS: Created labels array with shape {labels_array.shape} and {label_id-1} labels" + f"🔬 NAPARI PROCESS: Created labels array with shape {labels_array.shape} and {label_id - 1} labels" ) return labels_array @@ -1321,7 +1315,7 @@ def display_image(self, image_data: np.ndarray, metadata: dict) -> None: def process_image_message(self, message: bytes): """ - Process incoming image data message. + Process incoming image data message and send reply for REP socket. Args: message: Raw ZMQ message containing image data @@ -1338,13 +1332,15 @@ def process_image_message(self, message: bytes): # Handle batch of images/shapes images = data.get("images", []) display_config_dict = data.get("display_config") - + # Extract component names metadata for dimension labels (e.g., channel names) component_names_metadata = data.get("component_names_metadata", {}) if component_names_metadata: # Update server's component metadata cache self.component_metadata.update(component_names_metadata) - logger.info(f"🔬 NAPARI PROCESS: Updated component metadata: {list(component_names_metadata.keys())}") + logger.info( + f"🔬 NAPARI PROCESS: Updated component metadata: {list(component_names_metadata.keys())}" + ) for image_info in images: self._process_single_image(image_info, display_config_dict) @@ -1353,6 +1349,13 @@ def process_image_message(self, message: bytes): # Handle single image (legacy) self._process_single_image(data, data.get("display_config")) + # Send reply on REP socket (required pattern) + try: + reply = {"status": "success", "type": msg_type} + self.data_socket.send_json(reply) + except Exception as e: + logger.error(f"🔬 NAPARI PROCESS: Failed to send reply: {e}") + def _process_single_image( self, image_info: Dict[str, Any], display_config_dict: Dict[str, Any] ): @@ -1361,7 +1364,9 @@ def _process_single_image( path = image_info.get("path", "unknown") image_id = image_info.get("image_id") # UUID for acknowledgment - data_type = image_info.get("data_type", "image") # 'image', 'shapes', or 'points' + data_type = image_info.get( + "data_type", "image" + ) # 'image', 'shapes', or 'points' component_metadata = image_info.get("metadata", {}) # Log incoming metadata to debug well filtering issues @@ -1463,7 +1468,7 @@ def _process_single_image( except Exception as e: logger.error( f"🔬 NAPARI PROCESS: Failed to process {data_type} {path}: {e}", - exc_info=True + exc_info=True, ) if image_id: self._send_ack(image_id, status="error", error=str(e)) @@ -1511,10 +1516,10 @@ def _napari_viewer_process( # Set up dimension label tracking for well names # This will be populated as metadata arrives and used to update text overlay server.dimension_labels = {} # layer_key -> {component: [label1, label2, ...]} - + # Enable text overlay for dimension labels viewer.text_overlay.visible = True - viewer.text_overlay.color = 'white' + viewer.text_overlay.color = "white" viewer.text_overlay.font_size = 14 logger.info( @@ -1568,15 +1573,14 @@ def process_messages(): server.process_messages() # Process data messages (images) if ready + # REP socket requires recv->send alternation, so process one at a time if server._ready: - # Process multiple messages per timer tick for better throughput - for _ in range(10): # Process up to 10 messages per tick - try: - message = server.data_socket.recv(zmq.NOBLOCK) - server.process_image_message(message) - except zmq.Again: - # No more messages available - break + try: + message = server.data_socket.recv(zmq.NOBLOCK) + server.process_image_message(message) + except zmq.Again: + # No message available + pass # Connect timer to message processing timer.timeout.connect(process_messages) @@ -1747,7 +1751,9 @@ def __init__( replace_layers # If True, replace existing layers; if False, add new layers ) self.display_config = display_config # Configuration for display behavior - self.transport_mode = coerce_transport_mode(transport_mode) or ZMQTransportMode.IPC # ZMQ transport mode (IPC or TCP) + self.transport_mode = ( + coerce_transport_mode(transport_mode) or ZMQTransportMode.IPC + ) # ZMQ transport mode (IPC or TCP) self.process: Optional[multiprocessing.Process] = None self.zmq_context: Optional[zmq.Context] = None self.zmq_socket: Optional[zmq.Socket] = None @@ -1884,7 +1890,9 @@ def _start_viewer_sync(self): with self._lock: # Check if there's already a napari viewer running on the configured port - port_in_use = is_port_in_use(self.port, self.transport_mode, config=OPENHCS_ZMQ_CONFIG) + port_in_use = is_port_in_use( + self.port, self.transport_mode, config=OPENHCS_ZMQ_CONFIG + ) logger.info(f"🔬 VISUALIZER: Port {self.port} in use: {port_in_use}") if port_in_use: @@ -1942,10 +1950,9 @@ def _start_viewer_sync(self): _global_viewer_process = self.process _global_viewer_port = self.port - # Wait for napari viewer to be ready before setting up ZMQ - self._wait_for_viewer_ready() - - # Set up ZeroMQ client + # Set up ZeroMQ client immediately after process spawn. + # Readiness is owned by ViewerStateManager.wait_for_ready() to avoid + # duplicate/competing wait loops with conflicting timeout behavior. self._setup_zmq_client() # Check if process is running (different methods for subprocess vs multiprocessing) @@ -2260,9 +2267,7 @@ def _prepare_data_for_display( slicer[-1] = cpu_tensor.shape[-3] // 2 # Middle Z try: display_data = cpu_tensor[tuple(slicer)] - except ( - IndexError - ): # Handle cases where slicing might fail (e.g. very small dimensions) + except IndexError: # Handle cases where slicing might fail (e.g. very small dimensions) logger.error( f"Slicing failed for tensor with shape {cpu_tensor.shape} for step '{step_id_for_log}'.", exc_info=True, @@ -2295,7 +2300,6 @@ def _prepare_data_for_display( ) return None - def get_launch_command(self) -> list[str]: import os import sys diff --git a/openhcs/runtime/napari_viewer_server.py b/openhcs/runtime/napari_viewer_server.py new file mode 100644 index 000000000..8091ac2d9 --- /dev/null +++ b/openhcs/runtime/napari_viewer_server.py @@ -0,0 +1,1706 @@ +""" +Napari-based real-time visualization module. + +This module provides the NapariStreamVisualizer class for real-time +visualization of tensors during pipeline execution. +""" + +import logging +import multiprocessing +import os +import pickle +import subprocess +import sys +import threading +import time +import zmq +import numpy as np +from typing import Any, Dict, Optional +from qtpy.QtCore import QTimer + +from zmqruntime.config import TransportMode, ZMQConfig +from polystore.streaming_constants import StreamingDataType +from polystore.streaming.receivers.napari import ( + normalize_component_layout, + build_layer_key, +) +from zmqruntime.streaming import StreamingVisualizerServer, VisualizerProcessManager +from zmqruntime.transport import ( + coerce_transport_mode, + get_control_url, + get_zmq_transport_url, + is_port_in_use, + ping_control_port, + wait_for_server_ready, +) + +# Optional napari import - this module should only be imported if napari is available +try: + import napari +except ImportError: + napari = None + +if napari is None: + raise ImportError( + "napari is required for NapariStreamVisualizer. " + "Install it with: pip install 'openhcs[viz]' or pip install napari" + ) + + +logger = logging.getLogger(__name__) + +# ZMQ connection delay (ms) +ZMQ_CONNECTION_DELAY_MS = 100 # Brief delay for ZMQ connection to establish + +# Global process management for napari viewer +_global_viewer_process: Optional[multiprocessing.Process] = None +_global_viewer_port: Optional[int] = None +_global_process_lock = threading.Lock() + +# Registry of data type handlers (will be populated after helper functions are defined) +_DATA_TYPE_HANDLERS = None + + +def _cleanup_global_viewer() -> None: + """ + Clean up global napari viewer process for test mode. + + This forcibly terminates the napari viewer process to allow pytest to exit. + Should only be called in test mode. + """ + global _global_viewer_process + + with _global_process_lock: + if _global_viewer_process and _global_viewer_process.is_alive(): + logger.info("🔬 VISUALIZER: Terminating napari viewer for test cleanup") + _global_viewer_process.terminate() + _global_viewer_process.join(timeout=3) + + if _global_viewer_process.is_alive(): + logger.warning("🔬 VISUALIZER: Force killing napari viewer process") + _global_viewer_process.kill() + _global_viewer_process.join(timeout=1) + + _global_viewer_process = None + + +register_cleanup_callback(_cleanup_global_viewer) + + +def _parse_component_info_from_path(path_str: str): + """ + Fallback component parsing from path (used when component metadata unavailable). + + Args: + path_str: Path string like 'step_name/A01/s1_c2_z3.tif' + + Returns: + Dict with basic component info extracted from filename + """ + try: + import os + import re + + filename = os.path.basename(path_str) + + # Basic regex for common patterns + pattern = r"(?:s(\d+))?(?:_c(\d+))?(?:_z(\d+))?" + match = re.search(pattern, filename) + + components = {} + if match: + site, channel, z_index = match.groups() + if site: + components["site"] = site + if channel: + components["channel"] = channel + if z_index: + components["z_index"] = z_index + + return components + except Exception: + return {} + + +def _build_nd_shapes(layer_items, stack_components): + """ + Build nD shapes by prepending stack component indices to 2D shape coordinates. + + Args: + layer_items: List of items with 'data' (shapes_data) and 'components' + stack_components: List of component names to stack + + Returns: + Tuple of (all_shapes_nd, all_shape_types, all_properties) + """ + from polystore.roi_converters import NapariROIConverter + + all_shapes_nd = [] + all_shape_types = [] + all_properties = {"label": [], "area": [], "centroid_y": [], "centroid_x": []} + + # Build component value to index mapping (same as _build_nd_image_array) + component_values = {} + for comp in stack_components: + values = sorted(set(item["components"].get(comp, 0) for item in layer_items)) + component_values[comp] = values + + for item in layer_items: + shapes_data = item["data"] # List of shape dicts + components = item["components"] + + # Get stack component INDICES to prepend (not values!) + prepend_dims = [ + component_values[comp].index(components.get(comp, 0)) + for comp in stack_components + ] + + # Convert each shape to nD + for shape_dict in shapes_data: + # Use registry-based dimension handler + nd_coords = NapariROIConverter.add_dimensions_to_shape( + shape_dict, prepend_dims + ) + all_shapes_nd.append(nd_coords) + all_shape_types.append(shape_dict["type"]) + + # Extract properties + metadata = shape_dict.get("metadata", {}) + centroid = metadata.get("centroid", (0, 0)) + all_properties["label"].append(metadata.get("label", "")) + all_properties["area"].append(metadata.get("area", 0)) + all_properties["centroid_y"].append(centroid[0]) + all_properties["centroid_x"].append(centroid[1]) + + return all_shapes_nd, all_shape_types, all_properties + + +def _build_nd_points(layer_items, stack_components, component_values=None): + """ + Build nD points by prepending stack component indices to 2D point coordinates. + + Args: + layer_items: List of items with 'data' (list of point coordinate arrays) and 'components' + stack_components: List of component names to stack + component_values: Optional dict of {component: [sorted values]} to use for mapping. + If provided, uses this for building the stack dimensions. + If None, derives from layer_items. + + Returns: + Tuple of (all_points_nd, all_properties) + """ + all_points_nd = [] + all_properties = {"label": [], "component": []} + + # Build component value to index mapping (use global if provided) + if component_values is None: + component_values = {} + for comp in stack_components: + values = sorted( + set(item["components"].get(comp, 0) for item in layer_items) + ) + component_values[comp] = values + + for item in layer_items: + points_data = item["data"] # List of shape dicts from ROI converter + components = item["components"] + + # DEBUG: Log what we actually have + logger.info(f"🐛 DEBUG: points_data type: {type(points_data)}") + if isinstance(points_data, list) and len(points_data) > 0: + logger.info(f"🐛 DEBUG: first element type: {type(points_data[0])}") + logger.info(f"🐛 DEBUG: first element: {points_data[0]}") + + # Get stack component INDICES to prepend + prepend_dims = [ + component_values[comp].index(components.get(comp, 0)) + for comp in stack_components + ] + + # Convert each shape dict to nD points + # points_data is a list of dicts with 'type', 'coordinates', 'metadata' + for shape_dict in points_data: + # Only process 'points' type entries + if shape_dict.get("type") != "points": + continue + + coordinates = shape_dict.get("coordinates", []) + metadata = shape_dict.get("metadata", {}) + + # coordinates is a list of [y, x] pairs + # Prepend stack dimensions to each point: [y, x] -> [stack_idx, ..., y, x] + for coord in coordinates: + nd_coord = prepend_dims + list(coord) + all_points_nd.append(nd_coord) + + # Track properties for this point + all_properties["label"].append(metadata.get("label", "")) + all_properties["component"].append(metadata.get("component", 0)) + + return np.array(all_points_nd) if all_points_nd else np.empty( + (0, 2 + len(stack_components)) + ), all_properties + + +def _build_nd_image_array(layer_items, stack_components, component_values=None): + """ + Build nD image array by stacking images along stack component dimensions. + + Args: + layer_items: List of items with 'data' (image arrays) and 'components' + stack_components: List of component names to stack + component_values: Optional dict of {component: [sorted values]} to use for mapping. + If provided, uses this for building the stack dimensions. + If None, derives from layer_items (old behavior). + + Returns: + np.ndarray: Stacked image array + """ + # When component_values is provided (global tracker), always build multi-dimensional array + # This ensures ROIs at non-first indices get proper stack dimensions immediately + if component_values is not None: + # Using global component values - build proper multi-dimensional array + # even if we only have one item currently + pass # Fall through to multi-dimensional logic below + elif len(stack_components) == 1 and len(layer_items) > 1: + # Old behavior: Single stack component with multiple items - simple 3D stack + image_stack = [img["data"] for img in layer_items] + from openhcs.core.memory import stack_slices + + return stack_slices(image_stack, memory_type="numpy", gpu_id=0) + elif len(stack_components) == 1 and len(layer_items) == 1: + # Single item, single component, no global values - just return as-is + # (Will be wrapped in extra dimension if needed by caller) + return layer_items[0]["data"] + + # Multiple stack components OR using global component values - create multi-dimensional array + if component_values is None: + # Derive from layer items (old behavior when no global tracker) + component_values = {} + for comp in stack_components: + values = sorted(set(img["components"].get(comp, 0) for img in layer_items)) + component_values[comp] = values + + # Log component values for debugging + logger.info( + f"🔬 NAPARI PROCESS: Building nD array with stack_components={stack_components}, component_values={component_values}" + ) + + # Create empty array with shape (comp1_size, comp2_size, ..., y, x) + first_img = layer_items[0]["data"] + stack_shape = ( + tuple(len(component_values[comp]) for comp in stack_components) + + first_img.shape + ) + stacked_array = np.zeros(stack_shape, dtype=first_img.dtype) + logger.info( + f"🔬 NAPARI PROCESS: Created nD array with shape {stack_shape} from {len(layer_items)} items" + ) + + # Fill array + for img in layer_items: + # Get indices for this image + indices = tuple( + component_values[comp].index(img["components"].get(comp, 0)) + for comp in stack_components + ) + logger.debug( + f"🔬 NAPARI PROCESS: Placing image at indices {indices}, components={img['components']}" + ) + stacked_array[indices] = img["data"] + + return stacked_array + + +def _create_or_update_layer( + viewer, layers, layer_name, layer_type, data, **layer_kwargs +): + """ + Create or update a Napari layer of any type. + + All layers are handled identically: if layer exists, remove and recreate. + This ensures consistent behavior across all layer types. + + Args: + viewer: Napari viewer + layers: Dict of existing layers + layer_name: Name for the layer + layer_type: Type of layer ('image', 'shapes', 'points', etc.) + data: Data for the layer (format depends on layer_type) + **layer_kwargs: Additional kwargs to pass to viewer.add_() + + Returns: + The created layer + """ + # Check if layer exists + existing_layer = None + for layer in viewer.layers: + if layer.name == layer_name: + existing_layer = layer + break + + # Remove existing layer if present + if existing_layer is not None: + viewer.layers.remove(existing_layer) + layers.pop(layer_name, None) + logger.info( + f"🔬 NAPARI PROCESS: Removed existing {layer_type} layer {layer_name} for recreation" + ) + + # Get the add_* method for this layer type and create new layer + add_method = getattr(viewer, f"add_{layer_type}") + new_layer = add_method(data, name=layer_name, **layer_kwargs) + layers[layer_name] = new_layer + + # Log with appropriate count/info + if layer_type == "shapes": + count = len(data) if hasattr(data, "__len__") else 0 + logger.info( + f"🔬 NAPARI PROCESS: Created {layer_type} layer {layer_name} with {count} shapes" + ) + elif layer_type == "points": + count = len(data) if hasattr(data, "__len__") else 0 + logger.info( + f"🔬 NAPARI PROCESS: Created {layer_type} layer {layer_name} with {count} points" + ) + else: + logger.info(f"🔬 NAPARI PROCESS: Created {layer_type} layer {layer_name}") + + return new_layer + + +# Convenience wrappers that call the unified function +def _create_or_update_image_layer( + viewer, layers, layer_name, image_data, colormap, axis_labels=None +): + """Create or update a Napari image layer.""" + layer = _create_or_update_layer( + viewer, layers, layer_name, "image", image_data, colormap=colormap or "gray" + ) + # Set axis labels on viewer.dims (add_image axis_labels parameter doesn't work) + if axis_labels is not None: + viewer.dims.axis_labels = axis_labels + logger.info(f"🔬 NAPARI PROCESS: Set viewer.dims.axis_labels={axis_labels}") + return layer + + +def _create_or_update_shapes_layer( + viewer, layers, layer_name, shapes_data, shape_types, properties +): + """Create or update a Napari shapes layer.""" + return _create_or_update_layer( + viewer, + layers, + layer_name, + "shapes", + shapes_data, + shape_type=shape_types, + properties=properties, + edge_color="red", + face_color="transparent", + edge_width=2, + ) + + +def _create_or_update_points_layer(viewer, layers, layer_name, points_data, properties): + """Create or update a Napari points layer.""" + return _create_or_update_layer( + viewer, + layers, + layer_name, + "points", + points_data, + properties=properties, + face_color="green", + size=3, + ) + + +# Populate registry now that helper functions are defined +from polystore.streaming_constants import StreamingDataType + +_DATA_TYPE_HANDLERS = { + StreamingDataType.IMAGE: { + "build_nd_data": _build_nd_image_array, + "create_layer": _create_or_update_image_layer, + }, + StreamingDataType.SHAPES: { + "build_nd_data": _build_nd_shapes, + "create_layer": _create_or_update_shapes_layer, + }, + StreamingDataType.POINTS: { + "build_nd_data": _build_nd_points, + "create_layer": _create_or_update_points_layer, + }, +} + + +def _handle_component_aware_display( + viewer, + layers, + component_groups, + data, + path, + colormap, + display_config, + replace_layers, + component_metadata=None, + data_type="image", + server=None, +): + """ + Handle component-aware display following OpenHCS stacking patterns. + + Components marked as SLICE create separate layers, components marked as STACK are stacked together. + Layer naming follows canonical component order from display config. + + Args: + data_type: 'image' for image data, 'shapes' for ROI/shapes data (string or StreamingDataType enum) + server: NapariViewerServer instance (needed for debounced updates) + """ + try: + # Convert data_type to enum if needed (for backwards compatibility) + if isinstance(data_type, str): + data_type = StreamingDataType(data_type) + + # Use component metadata from ZMQ message - fail loud if not available + if not component_metadata: + raise ValueError(f"No component metadata available for path: {path}") + component_info = component_metadata + + component_modes, component_order = normalize_component_layout(display_config) + layer_key = build_layer_key( + component_info=component_info, + component_modes=component_modes, + component_order=component_order, + data_type=data_type, + ) + + # Log component modes for debugging + logger.info( + f"🔍 NAPARI PROCESS: component_modes={component_modes}, layer_key='{layer_key}'" + ) + + # Log layer key and component info for debugging + logger.info( + f"🔍 NAPARI PROCESS: layer_key='{layer_key}', component_info={component_info}" + ) + + # Reconcile cached layer/group state with live napari viewer after possible manual deletions + # CRITICAL: Only purge if the layer WAS in our cache but is now missing from viewer + # (user manually deleted it). Do NOT purge if layer was never created yet (debounced update pending). + try: + current_layer_names = {l.name for l in viewer.layers} + if layer_key not in current_layer_names and layer_key in layers: + # Layer was in our cache but is now missing from viewer - user deleted it + # Drop stale references so we will recreate the layer + num_items = len(component_groups.get(layer_key, [])) + layers.pop(layer_key, None) + component_groups.pop(layer_key, None) + logger.info( + f"🔬 NAPARI PROCESS: Reconciling state — '{layer_key}' was deleted from viewer; purged stale caches (had {num_items} items in component_groups)" + ) + except Exception: + # Fail-loud elsewhere; reconciliation is best-effort and must not mask display + pass + + # Initialize layer group if needed + if layer_key not in component_groups: + component_groups[layer_key] = [] + + # Handle replace_layers mode: clear all items for this layer_key + if replace_layers and component_groups[layer_key]: + logger.info( + f"🔬 NAPARI PROCESS: replace_layers=True, clearing {len(component_groups[layer_key])} existing items from layer '{layer_key}'" + ) + component_groups[layer_key] = [] + + # Check if an item with the same component_info AND data_type already exists + # If so, replace it instead of appending (prevents accumulation across runs) + # CRITICAL: Must include 'well' in comparison even if it's in STACK mode, + # otherwise images from different wells with same channel/z/field will be treated as duplicates + # CRITICAL: Must also check data_type to prevent images and ROIs from being treated as duplicates + existing_index = None + for i, item in enumerate(component_groups[layer_key]): + # Compare ALL components including well AND data_type + if item["components"] == component_info and item["data_type"] == data_type: + logger.info( + f"🔬 NAPARI PROCESS: Found duplicate - component_info: {component_info}, data_type: {data_type} at index {i}" + ) + existing_index = i + break + + new_item = { + "data": data, + "components": component_info, + "path": str(path), + "data_type": data_type, + } + + if existing_index is not None: + # Replace existing item with same components and data type + old_data_type = component_groups[layer_key][existing_index]["data_type"] + component_groups[layer_key][existing_index] = new_item + logger.info( + f"🔬 NAPARI PROCESS: Replaced {old_data_type} item in component_groups[{layer_key}] at index {existing_index}, total items: {len(component_groups[layer_key])}" + ) + else: + # Add new item + component_groups[layer_key].append(new_item) + logger.info( + f"🔬 NAPARI PROCESS: Added {data_type} to component_groups[{layer_key}], now has {len(component_groups[layer_key])} items" + ) + + # Schedule debounced layer update instead of immediate update + # This prevents race conditions when multiple items arrive rapidly + if server is None: + raise ValueError("Server instance required for debounced updates") + logger.info( + f"🔬 NAPARI PROCESS: Scheduling debounced update for {layer_key} (data_type={data_type})" + ) + server._schedule_layer_update( + layer_key, data_type, component_modes, component_order + ) + + except Exception as e: + import traceback + + logger.error( + f"🔬 NAPARI PROCESS: Component-aware display failed for {path}: {e}" + ) + logger.error( + f"🔬 NAPARI PROCESS: Component-aware display traceback: {traceback.format_exc()}" + ) + raise # Fail loud - no fallback + + +def _old_immediate_update_logic_removed(): + """ + Old immediate update logic removed in favor of debounced updates. + Kept as reference for the variable size handling logic that needs to be ported. + """ + pass + # Old code was here - removed to prevent race conditions + # Now using _schedule_layer_update -> _execute_layer_update -> _update_image_layer/_update_shapes_layer + + +class NapariViewerServer(StreamingVisualizerServer): + """ + ZMQ server for Napari viewer that receives images from clients. + + Inherits from ZMQServer ABC to get ping/pong, port management, etc. + Uses SUB socket to receive images from pipeline clients. + """ + + _server_type = "napari" # Registration key for AutoRegisterMeta + + def __init__( + self, + port: int, + viewer_title: str, + replace_layers: bool = False, + log_file_path: str = None, + transport_mode: OpenHCSTransportMode = OpenHCSTransportMode.IPC, + ): + """ + Initialize Napari viewer server. + + Args: + port: Data port for receiving images (control port will be port + 1000) + viewer_title: Title for the napari viewer window + replace_layers: If True, replace existing layers; if False, add new layers + log_file_path: Path to log file (for client discovery) + transport_mode: ZMQ transport mode (IPC or TCP) + """ + import zmq + + # Initialize with REP socket for receiving images (synchronous request/reply) + # REP socket forces workers to wait for acknowledgment before closing shared memory + super().__init__( + port, + viewer_type="napari", + host="*", + log_file_path=log_file_path, + data_socket_type=zmq.REP, + transport_mode=coerce_transport_mode(transport_mode), + config=OPENHCS_ZMQ_CONFIG, + ) + + self.viewer_title = viewer_title + self.replace_layers = replace_layers + self.viewer = None + self.layers = {} + self.component_groups = {} + self.dimension_labels = {} # Store dimension label mappings: layer_key -> {component: [labels]} + self.component_metadata = {} # Store component metadata from microscope handler: {component: {id: name}} + + # Global component value tracker for shared dimension mapping + # Maps tuple of stack_components -> {component: set of values} + # All layers with the same stack_components share the same global mapping + self.global_component_values = {} + + # Debouncing + locking for layer updates to prevent race conditions + import threading + + self.layer_update_lock = threading.Lock() # Prevent concurrent updates + self.pending_updates = {} # layer_key -> QTimer (debounce) + self.update_delay_ms = 1000 # Wait 200ms for more items before rebuilding + + # Batch processors for efficient accumulation and display + # Uses batch_size from config if available (defaults to None = wait for all) + self._batch_processors = {} # layer_key -> NapariBatchProcessor + self._batch_processor_lock = threading.Lock() + + # Ack socket handled by StreamingVisualizerServer + + def _setup_ack_socket(self): + """Setup PUSH socket for sending acknowledgments.""" + super()._setup_ack_socket() + + def _get_or_create_batch_processor( + self, layer_key: str, batch_size: Optional[int] = None + ): + """Get or create a batch processor for the given layer key.""" + with self._batch_processor_lock: + if layer_key not in self._batch_processors: + # Import here to avoid circular imports + from polystore.streaming.receivers.napari import NapariBatchProcessor + + processor = NapariBatchProcessor( + napari_server=self, + batch_size=batch_size, + debounce_delay_ms=self.update_delay_ms, + max_debounce_wait_ms=5000, # 5 second max wait + ) + self._batch_processors[layer_key] = processor + logger.info( + f"NapariViewerServer: Created batch processor for layer '{layer_key}' " + f"with batch_size={batch_size}" + ) + + return self._batch_processors[layer_key] + + def _update_global_component_values(self, stack_components, layer_items): + """ + Update the global component value tracker with values from new items. + + All layers sharing the same stack_components will use the same global mapping, + ensuring consistent component-to-index mapping across image and ROI layers. + + Args: + stack_components: Tuple/list of component names (e.g., ['channel', 'well']) + layer_items: List of items with 'components' dict + """ + # Use tuple as dict key (lists aren't hashable) + components_key = tuple(stack_components) + + # Initialize if needed + if components_key not in self.global_component_values: + self.global_component_values[components_key] = { + comp: set() for comp in stack_components + } + + # Add values from these items + global_values = self.global_component_values[components_key] + for item in layer_items: + for comp in stack_components: + value = item["components"].get(comp, 0) + global_values[comp].add(value) + + logger.info( + f"🔬 NAPARI PROCESS: Updated global component values for {stack_components}" + ) + for comp, values in global_values.items(): + sorted_values = sorted(values) + logger.info(f"🔬 NAPARI PROCESS: {comp}: {sorted_values}") + + def _get_global_component_values(self, stack_components): + """ + Get the global component values for a given set of stack components. + + For indexed components (channel, z_index, timepoint), expands to include + all values from 1 to max. For example, if only channel 2 is seen, returns [1, 2]. + This ensures proper stack dimensions even when some indices aren't present. + + Returns a dict of {component: sorted list of values} for all components + that have been seen across all layers sharing these stack components. + """ + components_key = tuple(stack_components) + + if components_key not in self.global_component_values: + return {comp: [] for comp in stack_components} + + # Convert sets to sorted lists and expand indexed components + global_values = self.global_component_values[components_key] + result = {} + + # Components that should be expanded from 1 to max (1-indexed) + INDEXED_COMPONENTS = {"channel", "z_index", "timepoint"} + + for comp, values in global_values.items(): + sorted_values = sorted(values) + + if comp in INDEXED_COMPONENTS and sorted_values: + # Expand to include all indices from 1 to max + # E.g., if we have [2, 4], expand to [1, 2, 3, 4] + max_value = max(sorted_values) + if max_value > 1: + # Create range from 1 to max_value (inclusive) + expanded_values = list(range(1, max_value + 1)) + result[comp] = expanded_values + logger.info( + f"🔬 NAPARI PROCESS: Expanded {comp} from {sorted_values} to {expanded_values}" + ) + else: + # Max is 1, no expansion needed + result[comp] = sorted_values + else: + # Non-indexed component (well, site, etc.) - use actual values + result[comp] = sorted_values + + return result + + def _schedule_layer_update( + self, layer_key, data_type, component_modes, component_order + ): + """ + Schedule a debounced layer update. + + Cancels any pending update for this layer and schedules a new one. + This prevents race conditions when multiple items arrive rapidly. + """ + # Cancel existing timer if any + if layer_key in self.pending_updates: + self.pending_updates[layer_key].stop() + logger.debug(f"🔬 NAPARI PROCESS: Cancelled pending update for {layer_key}") + + # Create new timer + timer = QTimer() + timer.setSingleShot(True) + timer.timeout.connect( + lambda: self._execute_layer_update( + layer_key, data_type, component_modes, component_order + ) + ) + timer.start(self.update_delay_ms) + self.pending_updates[layer_key] = timer + logger.debug( + f"🔬 NAPARI PROCESS: Scheduled update for {layer_key} in {self.update_delay_ms}ms" + ) + + def _execute_layer_update( + self, layer_key, data_type, component_modes, component_order + ): + """ + Execute the actual layer update after debounce delay. + + Uses batch processor for efficient accumulation and display. + """ + # Remove timer + self.pending_updates.pop(layer_key, None) + + # Get current items for this layer + layer_items = self.component_groups.get(layer_key, []) + if not layer_items: + logger.warning( + f"🔬 NAPARI PROCESS: No items found for {layer_key}, skipping update" + ) + return + + # Log layer composition + wells_in_layer = set( + item["components"].get("well", "unknown") for item in layer_items + ) + logger.info( + f"🔬 NAPARI PROCESS: layer_key='{layer_key}' has {len(layer_items)} items from wells: {sorted(wells_in_layer)}" + ) + + # Use batch processor for efficient accumulation and display + # This avoids recreating layers from scratch on every update + batch_processor = self._get_or_create_batch_processor(layer_key) + batch_processor.add_items( + layer_key=layer_key, + items=layer_items, + display_config={"component_modes": component_modes}, + component_names_metadata=self.component_metadata, + ) + + def _setup_dimension_label_handler(self, layer_key, stack_components): + """ + Set up event handler to update text overlay when dimensions change. + + This connects the viewer's dimension slider changes to text overlay updates, + displaying categorical labels (like well IDs) instead of numeric indices. + + Args: + layer_key: The layer to monitor for dimension changes + stack_components: List of components that are stacked (e.g., ['well', 'channel']) + """ + if not self.viewer or not stack_components: + return + + # Get dimension label mappings for this layer + layer_labels = self.dimension_labels.get(layer_key, {}) + if not layer_labels: + return + + def update_dimension_label(event=None): + """Update text overlay with current dimension labels.""" + try: + current_step = self.viewer.dims.current_step + + # Build label text from stacked components + label_parts = [] + for i, component in enumerate(stack_components): + if component in layer_labels: + labels = layer_labels[component] + # Get current index for this dimension + if i < len(current_step): + idx = current_step[i] + if 0 <= idx < len(labels): + label = labels[idx] + # Don't show if label is None or "None" + if label and str(label).lower() != "none": + label_parts.append(label) + + if label_parts: + self.viewer.text_overlay.text = " | ".join(label_parts) + else: + self.viewer.text_overlay.text = "" + + except Exception as e: + logger.debug(f"🔬 NAPARI PROCESS: Error updating dimension label: {e}") + + # Connect to dimension change events + try: + self.viewer.dims.events.current_step.connect(update_dimension_label) + # Initial update + update_dimension_label() + logger.info( + f"🔬 NAPARI PROCESS: Set up dimension label handler for {layer_key}" + ) + except Exception as e: + logger.warning( + f"🔬 NAPARI PROCESS: Failed to setup dimension label handler: {e}" + ) + + def _update_image_layer( + self, layer_key, layer_items, stack_components, component_modes + ): + """Update an image layer with the current items.""" + + # Update global component tracker with values from these items + self._update_global_component_values(stack_components, layer_items) + + # Get global component values (union of all layers with same stack_components) + global_component_values = self._get_global_component_values(stack_components) + + # Check if images have different shapes and pad if needed + shapes = [item["data"].shape for item in layer_items] + if len(set(shapes)) > 1: + logger.info( + f"🔬 NAPARI PROCESS: Images in layer {layer_key} have different shapes - padding to max size" + ) + + # Find max dimensions + first_shape = shapes[0] + max_shape = list(first_shape) + for img_shape in shapes: + for i, dim in enumerate(img_shape): + max_shape[i] = max(max_shape[i], dim) + max_shape = tuple(max_shape) + + # Pad all images to max shape + for img_info in layer_items: + img_data = img_info["data"] + if img_data.shape != max_shape: + # Calculate padding for each dimension + pad_width = [] + for i, (current_dim, max_dim) in enumerate( + zip(img_data.shape, max_shape) + ): + pad_before = 0 + pad_after = max_dim - current_dim + pad_width.append((pad_before, pad_after)) + + # Pad with zeros + padded_data = np.pad( + img_data, pad_width, mode="constant", constant_values=0 + ) + img_info["data"] = padded_data + logger.debug( + f"🔬 NAPARI PROCESS: Padded image from {img_data.shape} to {padded_data.shape}" + ) + + logger.info( + f"🔬 NAPARI PROCESS: Building nD data for {layer_key} from {len(layer_items)} items" + ) + stacked_data = _build_nd_image_array( + layer_items, stack_components, global_component_values + ) + + # Determine colormap + colormap = None + if "channel" in component_modes and component_modes["channel"] == "slice": + first_item = layer_items[0] + channel_value = first_item["components"].get("channel") + if channel_value == 1: + colormap = "green" + elif channel_value == 2: + colormap = "red" + + # Build axis labels for stacked dimensions + # Format: (component1_name, component2_name, ..., 'y', 'x') + # The stack components appear in the same order as in stack_components list + # Must be a tuple for Napari + axis_labels = None + if stack_components: + axis_labels = tuple(list(stack_components) + ["y", "x"]) + logger.info( + f"🔬 NAPARI PROCESS: Built axis_labels={axis_labels} for stack_components={stack_components}" + ) + + # Build dimension labels from component values + # Use global component values to ensure consistency across all layers + dimension_labels = {} + + # Component abbreviation mapping + COMPONENT_ABBREV = { + "channel": "Ch", + "z_index": "Z", + "timepoint": "T", + "site": "Site", + "well": "Well", + } + + for comp in stack_components: + # Use global component values instead of just this layer's values + values = global_component_values[comp] + + # Try to get human-readable labels from metadata if available + labels = [] + + # Check if we have metadata for this component type + comp_metadata = self.component_metadata.get(comp, {}) + + for v in values: + # First try to get name from metadata (e.g., channel name) + metadata_name = comp_metadata.get(str(v)) + + if metadata_name and str(metadata_name).lower() != "none": + # Use metadata name with index for clarity + if comp == "channel": + labels.append(f"Ch{v}: {metadata_name}") + elif comp == "well": + labels.append( + f"{metadata_name}" + ) # Well names are already good (e.g., "A01") + else: + labels.append(f"{comp.title()} {v}: {metadata_name}") + else: + # No metadata - use abbreviated component name + index + abbrev = COMPONENT_ABBREV.get(comp, comp) + labels.append(f"{abbrev} {v}") + + dimension_labels[comp] = labels + + # Store dimension labels for this layer + self.dimension_labels[layer_key] = dimension_labels + + # Create or update the layer + _create_or_update_image_layer( + self.viewer, self.layers, layer_key, stacked_data, colormap, axis_labels + ) + + # Set up dimension label handler (connects dimension changes to text overlay) + self._setup_dimension_label_handler(layer_key, stack_components) + + def _update_shapes_layer( + self, layer_key, layer_items, stack_components, component_modes + ): + """Update a shapes layer - use labels instead of shapes for efficiency.""" + logger.info( + f"🔬 NAPARI PROCESS: Converting shapes to labels for {layer_key} from {len(layer_items)} items" + ) + + # Update global component tracker with values from these items + self._update_global_component_values(stack_components, layer_items) + + # Get global component values (union of all layers with same stack_components) + global_component_values = self._get_global_component_values(stack_components) + + # Convert shapes to label masks (much faster than individual shapes) + # This happens synchronously but is fast because we're just creating arrays + labels_data = self._shapes_to_labels( + layer_items, stack_components, global_component_values + ) + + # Remove existing layer if it exists + if layer_key in self.layers: + try: + self.viewer.layers.remove(self.layers[layer_key]) + logger.info( + f"🔬 NAPARI PROCESS: Removed existing labels layer {layer_key} for recreation" + ) + except Exception as e: + logger.warning( + f"Failed to remove existing labels layer {layer_key}: {e}" + ) + + # Create new labels layer + new_layer = self.viewer.add_labels(labels_data, name=layer_key) + self.layers[layer_key] = new_layer + logger.info( + f"🔬 NAPARI PROCESS: Created labels layer {layer_key} with shape {labels_data.shape}" + ) + + def _update_points_layer( + self, layer_key, layer_items, stack_components, component_modes + ): + """Update a points layer (for skeleton tracings and point-based ROIs).""" + # Filter to only POINTS items (exclude IMAGE items that may share the same layer_key) + points_items = [ + item + for item in layer_items + if item.get("data_type") == StreamingDataType.POINTS + ] + + if not points_items: + logger.warning( + f"🔬 NAPARI PROCESS: No POINTS items found for {layer_key}, skipping" + ) + return + + logger.info( + f"🔬 NAPARI PROCESS: Building points layer for {layer_key} from {len(points_items)} items (filtered from {len(layer_items)} total)" + ) + + # Update global component tracker with ALL items (images + points) to stay in sync + self._update_global_component_values(stack_components, layer_items) + + # Get global component values (union of all layers with same stack_components) + global_component_values = self._get_global_component_values(stack_components) + + # Build nD points data using ONLY the points items BUT with global component values + points_data, properties = _build_nd_points( + points_items, stack_components, global_component_values + ) + + # Create or update the points layer + _create_or_update_points_layer( + self.viewer, self.layers, layer_key, points_data, properties + ) + + logger.info( + f"🔬 NAPARI PROCESS: Created points layer {layer_key} with {len(points_data)} points" + ) + + def _shapes_to_labels(self, layer_items, stack_components, component_values): + """Convert shapes data to label masks. + + Args: + layer_items: List of shape items to convert + stack_components: List of component names for stack dimensions + component_values: Dict of {component: [sorted values]} from global tracker + """ + from skimage import draw + + # Use global component values passed in + # This ensures ROIs and images share the same component-to-index mapping + logger.info( + f"🔬 NAPARI PROCESS: Building ROI stack with global component values" + ) + for comp, values in component_values.items(): + logger.info(f"🔬 NAPARI PROCESS: {comp}: {values}") + + # Determine output shape + # Get image shape from first item's shapes data + first_shapes = layer_items[0]["data"] + if not first_shapes: + # No shapes, return empty array with reasonable default size + logger.warning( + "🔬 NAPARI PROCESS: No shapes data, creating default 512x512 array" + ) + return np.zeros((1, 1, 512, 512), dtype=np.uint16) + + # Estimate image size from shape coordinates + max_y, max_x = 0, 0 + for shape_dict in first_shapes: + if shape_dict["type"] == "polygon": + coords = np.array(shape_dict["coordinates"]) + max_y = max(max_y, int(np.max(coords[:, 0])) + 1) + max_x = max(max_x, int(np.max(coords[:, 1])) + 1) + elif shape_dict["type"] == "path": + # Handle path (polyline) type - get bounding box + coords = np.array(shape_dict["coordinates"]) + if len(coords) > 0: + max_y = max(max_y, int(np.max(coords[:, 0])) + 1) + max_x = max(max_x, int(np.max(coords[:, 1])) + 1) + elif shape_dict["type"] == "points": + # Handle points type - get bounding box + coords = np.array(shape_dict["coordinates"]) + if len(coords) > 0: + max_y = max(max_y, int(np.max(coords[:, 0])) + 1) + max_x = max(max_x, int(np.max(coords[:, 1])) + 1) + + # Ensure minimum valid dimensions (avoid 0x0 shapes) + if max_y == 0 or max_x == 0: + logger.warning( + f"🔬 NAPARI PROCESS: Invalid shape dimensions (y={max_y}, x={max_x}), using default 512x512" + ) + max_y = max(max_y, 512) + max_x = max(max_x, 512) + + # Build nD shape + nd_shape = [] + for comp in stack_components: + nd_shape.append(len(component_values[comp])) + nd_shape.extend([max_y, max_x]) + + # Create empty label array + labels_array = np.zeros(nd_shape, dtype=np.uint16) + + # Fill in labels for each item + label_id = 1 + for item in layer_items: + # Get indices for this item + indices = [] + for comp in stack_components: + comp_value = item["components"].get(comp, 0) + idx = component_values[comp].index(comp_value) + indices.append(idx) + + # Get shapes data + shapes_data = item["data"] + + # Draw each shape into the label mask + for shape_dict in shapes_data: + if shape_dict["type"] == "polygon": + coords = np.array(shape_dict["coordinates"]) + rr, cc = draw.polygon( + coords[:, 0], coords[:, 1], shape=labels_array.shape[-2:] + ) + + # Set label at the correct nD position + full_indices = tuple(indices) + (rr, cc) + labels_array[full_indices] = label_id + label_id += 1 + + elif shape_dict["type"] == "path": + # Draw polyline (skeleton branches) + # Skeleton paths from skan already contain every pixel in the branch, + # so we just set the label at each coordinate directly (no line drawing needed) + coords = np.array(shape_dict["coordinates"]) + if len(coords) >= 1: + # Extract row and column indices + rr = coords[:, 0].astype(int) + cc = coords[:, 1].astype(int) + + # Clip to image bounds + valid = ( + (rr >= 0) + & (rr < labels_array.shape[-2]) + & (cc >= 0) + & (cc < labels_array.shape[-1]) + ) + rr, cc = rr[valid], cc[valid] + + # Set label at the correct nD position + full_indices = tuple(indices) + (rr, cc) + labels_array[full_indices] = label_id + label_id += 1 + + logger.info( + f"🔬 NAPARI PROCESS: Created labels array with shape {labels_array.shape} and {label_id - 1} labels" + ) + return labels_array + + def _send_ack(self, image_id: str, status: str = "success", error: str = None): + """Send acknowledgment that an image was processed. + + Args: + image_id: UUID of the processed image + status: 'success' or 'error' + error: Error message if status='error' + """ + self.send_ack(image_id, status=status, error=error) + + def _create_pong_response(self) -> Dict[str, Any]: + """Override to add Napari-specific fields and memory usage.""" + response = super()._create_pong_response() + response["viewer"] = "napari" + response["openhcs"] = True + response["server"] = "NapariViewer" + + # Add memory usage + try: + import psutil + import os + + process = psutil.Process(os.getpid()) + response["memory_mb"] = process.memory_info().rss / 1024 / 1024 + response["cpu_percent"] = process.cpu_percent(interval=0) + except Exception: + pass + + return response + + def handle_control_message(self, message: Dict[str, Any]) -> Dict[str, Any]: + """ + Handle control messages beyond ping/pong. + + Supported message types: + - shutdown: Graceful shutdown (closes viewer) + - force_shutdown: Force shutdown (same as shutdown for Napari) + - clear_state: Clear accumulated component groups (for new pipeline runs) + """ + msg_type = message.get("type") + + if msg_type == "shutdown" or msg_type == "force_shutdown": + logger.info(f"🔬 NAPARI SERVER: {msg_type} requested, closing viewer") + self.request_shutdown() + + # Schedule viewer close on Qt event loop to trigger application exit + # This must be done after sending the response, so we use QTimer.singleShot + if self.viewer is not None: + from qtpy import QtCore + + QtCore.QTimer.singleShot(100, self.viewer.close) + + return { + "type": "shutdown_ack", + "status": "success", + "message": "Napari viewer shutting down", + } + + elif msg_type == "clear_state": + # Clear accumulated component groups to prevent shape accumulation across runs + logger.info( + f"🔬 NAPARI SERVER: Clearing component groups (had {len(self.component_groups)} groups)" + ) + self.component_groups.clear() + return { + "type": "clear_state_ack", + "status": "success", + "message": "Component groups cleared", + } + + # Unknown message type + return {"status": "ok"} + + def handle_data_message(self, message: Dict[str, Any]): + """Handle incoming image data - called by process_messages().""" + # This will be called from the Qt timer + pass + + def display_image(self, image_data: np.ndarray, metadata: dict) -> None: + """Display a single image payload (best-effort helper).""" + image_info = { + "data": image_data, + "shape": getattr(image_data, "shape", None), + "dtype": getattr(image_data, "dtype", None), + "metadata": metadata, + } + self._process_single_image(image_info, {}) + + def process_image_message(self, message: bytes): + """ + Process incoming image data message and send reply for REP socket. + + Args: + message: Raw ZMQ message containing image data + """ + import json + + # Parse JSON message + data = json.loads(message.decode("utf-8")) + + msg_type = data.get("type") + + # Check message type + if msg_type == "batch": + # Handle batch of images/shapes + images = data.get("images", []) + display_config_dict = data.get("display_config") + + # Extract component names metadata for dimension labels (e.g., channel names) + component_names_metadata = data.get("component_names_metadata", {}) + if component_names_metadata: + # Update server's component metadata cache + self.component_metadata.update(component_names_metadata) + logger.info( + f"🔬 NAPARI PROCESS: Updated component metadata: {list(component_names_metadata.keys())}" + ) + + for image_info in images: + self._process_single_image(image_info, display_config_dict) + + else: + # Handle single image (legacy) + self._process_single_image(data, data.get("display_config")) + + # Send reply on REP socket (required pattern) + try: + reply = {"status": "success", "type": msg_type} + self.data_socket.send_json(reply) + except Exception as e: + logger.error(f"🔬 NAPARI PROCESS: Failed to send reply: {e}") + + def _process_single_image( + self, image_info: Dict[str, Any], display_config_dict: Dict[str, Any] + ): + """Process a single image or shapes data and display in Napari.""" + import numpy as np + + path = image_info.get("path", "unknown") + image_id = image_info.get("image_id") # UUID for acknowledgment + data_type = image_info.get( + "data_type", "image" + ) # 'image', 'shapes', or 'points' + component_metadata = image_info.get("metadata", {}) + + # Log incoming metadata to debug well filtering issues + logger.info( + f"🔍 NAPARI PROCESS: Received {data_type} with metadata: {component_metadata} (path: {path})" + ) + + try: + # Check if this is shapes or points data + if data_type == "shapes" or data_type == "points": + # Handle shapes/ROIs/points - just pass the shapes data directly + shapes_data = image_info.get("shapes", []) + data = shapes_data + colormap = None # Shapes/points don't use colormap + else: + # Handle image data - load from shared memory or direct data + shape = image_info.get("shape") + dtype = image_info.get("dtype") + shm_name = image_info.get("shm_name") + direct_data = image_info.get("data") + + # Load image data + if shm_name: + from multiprocessing import shared_memory + + try: + shm = shared_memory.SharedMemory(name=shm_name) + data = np.ndarray(shape, dtype=dtype, buffer=shm.buf).copy() + shm.close() + # Unlink shared memory after copying - viewer is responsible for cleanup + try: + shm.unlink() + except FileNotFoundError: + # Already unlinked (race condition or duplicate message) + logger.debug( + f"🔬 NAPARI PROCESS: Shared memory {shm_name} already unlinked" + ) + except Exception as e: + logger.warning( + f"🔬 NAPARI PROCESS: Failed to unlink shared memory {shm_name}: {e}" + ) + except FileNotFoundError: + # Shared memory doesn't exist - likely already processed and unlinked + logger.error( + f"🔬 NAPARI PROCESS: Shared memory {shm_name} not found - may have been already processed" + ) + if image_id: + self._send_ack( + image_id, + status="error", + error=f"Shared memory {shm_name} not found", + ) + return + except Exception as e: + logger.error( + f"🔬 NAPARI PROCESS: Failed to open shared memory {shm_name}: {e}" + ) + if image_id: + self._send_ack( + image_id, + status="error", + error=f"Failed to open shared memory: {e}", + ) + raise + elif direct_data: + data = np.array(direct_data, dtype=dtype).reshape(shape) + else: + logger.warning("🔬 NAPARI PROCESS: No image data in message") + if image_id: + self._send_ack( + image_id, status="error", error="No image data in message" + ) + return + + # Extract colormap + colormap = "viridis" + if display_config_dict and "colormap" in display_config_dict: + colormap = display_config_dict["colormap"] + + # Component-aware layer management (handles both images and shapes) + _handle_component_aware_display( + self.viewer, + self.layers, + self.component_groups, + data, + path, + colormap, + display_config_dict or {}, + self.replace_layers, + component_metadata, + data_type, + server=self, + ) + + # Send acknowledgment that data was successfully displayed + if image_id: + self._send_ack(image_id, status="success") + + except Exception as e: + logger.error( + f"🔬 NAPARI PROCESS: Failed to process {data_type} {path}: {e}", + exc_info=True, + ) + if image_id: + self._send_ack(image_id, status="error", error=str(e)) + # Don't re-raise - continue processing other messages instead of crashing + + +def _napari_viewer_process( + port: int, + viewer_title: str, + replace_layers: bool = False, + log_file_path: str = None, + transport_mode: OpenHCSTransportMode = OpenHCSTransportMode.IPC, +): + """ + Napari viewer process entry point. Runs in a separate process. + Listens for ZeroMQ messages with image data to display. + + Args: + port: ZMQ port to listen on + viewer_title: Title for the napari viewer window + replace_layers: If True, replace existing layers; if False, add new layers with unique names + log_file_path: Path to log file (for client discovery via ping/pong) + transport_mode: ZMQ transport mode (IPC or TCP) + """ + try: + import zmq + import napari + + # Create ZMQ server instance (inherits from ZMQServer ABC) + server = NapariViewerServer( + port, viewer_title, replace_layers, log_file_path, transport_mode + ) + + # Start the server (binds sockets) + server.start() + + # Create napari viewer in this process (main thread) + viewer = napari.Viewer(title=viewer_title, show=True) + server.viewer = viewer + + # Initialize layers dictionary with existing layers (for reconnection scenarios) + for layer in viewer.layers: + server.layers[layer.name] = layer + + # Set up dimension label tracking for well names + # This will be populated as metadata arrives and used to update text overlay + server.dimension_labels = {} # layer_key -> {component: [label1, label2, ...]} + + # Enable text overlay for dimension labels + viewer.text_overlay.visible = True + viewer.text_overlay.color = "white" + viewer.text_overlay.font_size = 14 + + logger.info( + f"🔬 NAPARI PROCESS: Viewer started on data port {port}, control port {server.control_port}" + ) + + # Add cleanup handler for when viewer is closed + def cleanup_and_exit(): + logger.info("🔬 NAPARI PROCESS: Viewer closed, cleaning up and exiting...") + try: + server.stop() + except: + pass + sys.exit(0) + + # Connect the viewer close event to cleanup + viewer.window.qt_viewer.destroyed.connect(cleanup_and_exit) + + # Use proper Qt event loop integration + import sys + from qtpy import QtWidgets, QtCore + + # Ensure Qt platform is properly set for detached processes + import os + import platform + + if "QT_QPA_PLATFORM" not in os.environ: + if platform.system() == "Darwin": # macOS + os.environ["QT_QPA_PLATFORM"] = "cocoa" + elif platform.system() == "Linux": + os.environ["QT_QPA_PLATFORM"] = "xcb" + os.environ["QT_X11_NO_MITSHM"] = "1" + # Windows doesn't need QT_QPA_PLATFORM set + elif platform.system() == "Linux": + # Disable shared memory for X11 (helps with display issues in detached processes) + os.environ["QT_X11_NO_MITSHM"] = "1" + + # Get the Qt application + app = QtWidgets.QApplication.instance() + if app is None: + app = QtWidgets.QApplication(sys.argv) + + # Ensure the application DOES quit when the napari window closes + app.setQuitOnLastWindowClosed(True) + + # Set up a QTimer for message processing + timer = QtCore.QTimer() + + def process_messages(): + # Process control messages (ping/pong handled by ABC) + server.process_messages() + + # Process data messages (images) if ready + # REP socket requires recv->send alternation, so process one at a time + if server._ready: + try: + message = server.data_socket.recv(zmq.NOBLOCK) + server.process_image_message(message) + except zmq.Again: + # No message available + pass + + # Connect timer to message processing + timer.timeout.connect(process_messages) + timer.start(50) # Process messages every 50ms + + logger.info("🔬 NAPARI PROCESS: Starting Qt event loop") + + # Run the Qt event loop - this keeps napari responsive + app.exec_() + + except Exception as e: + logger.error(f"🔬 NAPARI PROCESS: Fatal error: {e}") + finally: + logger.info("🔬 NAPARI PROCESS: Shutting down") + if "server" in locals(): + server.stop() + + +def _spawn_detached_napari_process( + port: int, + viewer_title: str, + replace_layers: bool = False, + transport_mode: OpenHCSTransportMode = OpenHCSTransportMode.IPC, +) -> subprocess.Popen: + """ + Spawn a completely detached napari viewer process that survives parent termination. + + This creates a subprocess that runs independently and won't be terminated when + the parent process exits, enabling true persistence across pipeline runs. + + Args: + port: ZMQ port to listen on + viewer_title: Title for the napari viewer window + replace_layers: If True, replace existing layers; if False, add new layers + transport_mode: ZMQ transport mode (IPC or TCP) + """ + # Use a simpler approach: spawn python directly with the napari viewer module + # This avoids temporary file issues and import problems + + # Create the command to run the napari viewer directly + current_dir = os.getcwd() + python_code = f""" +import sys +import os + +# Detach from parent process group (Unix only) +if hasattr(os, "setsid"): + try: + os.setsid() + except OSError: + pass + +# Add current working directory to Python path +sys.path.insert(0, {repr(current_dir)}) + +try: + from openhcs.runtime.napari_stream_visualizer import _napari_viewer_process + from openhcs.core.config import TransportMode + transport_mode = TransportMode.{transport_mode.name} + _napari_viewer_process({port}, {repr(viewer_title)}, {replace_layers}, {repr(current_dir + "/.napari_log_path_placeholder")}, transport_mode) +except Exception as e: + import logging + logger = logging.getLogger("openhcs.runtime.napari_detached") + logger.error(f"Detached napari error: {{e}}") + import traceback + logger.error(traceback.format_exc()) + sys.exit(1) +""" + + try: + # Create log file for detached process + log_dir = os.path.expanduser("~/.local/share/openhcs/logs") + os.makedirs(log_dir, exist_ok=True) + log_file = os.path.join(log_dir, f"napari_detached_port_{port}.log") + + # Replace placeholder with actual log file path in python code + python_code = python_code.replace( + repr(current_dir + "/.napari_log_path_placeholder"), repr(log_file) + ) + + # Use subprocess.Popen with detachment flags + if sys.platform == "win32": + # Windows: Use CREATE_NEW_PROCESS_GROUP to detach but preserve display environment + env = os.environ.copy() # Preserve environment variables + with open(log_file, "w") as log_f: + process = subprocess.Popen( + [sys.executable, "-c", python_code], + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP + | subprocess.DETACHED_PROCESS, + env=env, + cwd=os.getcwd(), + stdout=log_f, + stderr=subprocess.STDOUT, + ) + else: + # Unix: Use start_new_session to detach but preserve display environment + env = os.environ.copy() # Preserve DISPLAY and other environment variables + + # Ensure Qt platform is set for GUI display + import platform + + if "QT_QPA_PLATFORM" not in env: + if platform.system() == "Darwin": # macOS + env["QT_QPA_PLATFORM"] = "cocoa" + elif platform.system() == "Linux": + env["QT_QPA_PLATFORM"] = "xcb" + env["QT_X11_NO_MITSHM"] = "1" + # Windows doesn't need QT_QPA_PLATFORM set + elif platform.system() == "Linux": + # Ensure Qt can find the display + env["QT_X11_NO_MITSHM"] = ( + "1" # Disable shared memory for X11 (helps with some display issues) + ) + + # Redirect stdout/stderr to log file for debugging + log_f = open(log_file, "w") + process = subprocess.Popen( + [sys.executable, "-c", python_code], + env=env, + cwd=os.getcwd(), + stdout=log_f, + stderr=subprocess.STDOUT, + start_new_session=True, # CRITICAL: Detach from parent process group + ) + + logger.info( + f"🔬 VISUALIZER: Detached napari process started (PID: {process.pid}), logging to {log_file}" + ) + return process + + except Exception as e: + logger.error(f"🔬 VISUALIZER: Failed to spawn detached napari process: {e}") + raise e diff --git a/openhcs/runtime/zmq_execution_client.py b/openhcs/runtime/zmq_execution_client.py index cd4e08af1..ef167f15a 100644 --- a/openhcs/runtime/zmq_execution_client.py +++ b/openhcs/runtime/zmq_execution_client.py @@ -1,6 +1,8 @@ """OpenHCS execution client built on zmqruntime ExecutionClient.""" + from __future__ import annotations +import hashlib import logging import subprocess import sys @@ -17,7 +19,7 @@ logger = logging.getLogger(__name__) -class OpenHCSExecutionClient(ExecutionClient): +class ZMQExecutionClient(ExecutionClient): """ZMQ client for OpenHCS pipeline execution with progress streaming.""" def __init__( @@ -47,41 +49,109 @@ def serialize_task(self, task: Any, config: Any = None) -> dict: global_config = task.get("global_config") pipeline_config = task.get("pipeline_config") config_params = task.get("config_params") + compile_only = task.get("compile_only", False) + compile_artifact_id = task.get("compile_artifact_id") pipeline_code = generate_python_source( Assignment("pipeline_steps", pipeline_steps), header="# Edit this pipeline and save to apply changes", clean_mode=True, ) - request = {"type": "execute", "plate_id": str(plate_id), "pipeline_code": pipeline_code} + request = { + "type": "execute", + "plate_id": str(plate_id), + "pipeline_code": pipeline_code, + } + pipeline_sha = hashlib.sha256(pipeline_code.encode("utf-8")).hexdigest()[:12] + if compile_only: + request["compile_only"] = True + if compile_artifact_id: + request["compile_artifact_id"] = str(compile_artifact_id) if config_params: request["config_params"] = config_params else: - request["config_code"] = generate_python_source( + config_code = generate_python_source( Assignment("config", global_config), header="# Configuration Code", clean_mode=True, ) + request["config_code"] = config_code + config_sha = hashlib.sha256(config_code.encode("utf-8")).hexdigest()[:12] if pipeline_config: - request["pipeline_config_code"] = generate_python_source( + pipeline_config_code = generate_python_source( Assignment("config", pipeline_config), header="# Configuration Code", clean_mode=True, ) + request["pipeline_config_code"] = pipeline_config_code + pipeline_config_sha = hashlib.sha256( + pipeline_config_code.encode("utf-8") + ).hexdigest()[:12] + else: + pipeline_config_sha = "-" + if config_params: + config_sha = "params" + pipeline_config_sha = "params" + logger.info( + "Serialize task: plate=%s compile_only=%s artifact_id=%s step_count=%s pipeline_sha=%s config_sha=%s pipeline_config_sha=%s", + plate_id, + bool(compile_only), + compile_artifact_id, + len(pipeline_steps) if isinstance(pipeline_steps, list) else "?", + pipeline_sha, + config_sha, + pipeline_config_sha, + ) return request - def submit_pipeline(self, plate_id, pipeline_steps, global_config, pipeline_config=None, config_params=None): + def submit_pipeline( + self, + plate_id, + pipeline_steps, + global_config, + pipeline_config=None, + config_params=None, + compile_artifact_id=None, + ): task = { "plate_id": plate_id, "pipeline_steps": pipeline_steps, "global_config": global_config, "pipeline_config": pipeline_config, "config_params": config_params, + "compile_artifact_id": compile_artifact_id, } return self.submit_execution(task) - def execute_pipeline(self, plate_id, pipeline_steps, global_config, pipeline_config=None, config_params=None): - response = self.submit_pipeline(plate_id, pipeline_steps, global_config, pipeline_config, config_params) + def submit_compile( + self, + plate_id, + pipeline_steps, + global_config, + pipeline_config=None, + config_params=None, + ): + task = { + "plate_id": plate_id, + "pipeline_steps": pipeline_steps, + "global_config": global_config, + "pipeline_config": pipeline_config, + "config_params": config_params, + "compile_only": True, + } + return self.submit_execution(task) + + def execute_pipeline( + self, + plate_id, + pipeline_steps, + global_config, + pipeline_config=None, + config_params=None, + ): + response = self.submit_pipeline( + plate_id, pipeline_steps, global_config, pipeline_config, config_params + ) if response.get("status") == "accepted": execution_id = response.get("execution_id") return self.wait_for_completion(execution_id) @@ -93,16 +163,38 @@ def get_status(self, execution_id=None): def _spawn_server_process(self): import os import glob + import logging log_dir = Path.home() / ".local" / "share" / "openhcs" / "logs" log_dir.mkdir(parents=True, exist_ok=True) - log_file_path = log_dir / f"openhcs_zmq_server_port_{self.port}_{int(time.time() * 1000000)}.log" - cmd = [sys.executable, "-m", "openhcs.runtime.zmq_execution_server_launcher", "--port", str(self.port)] + log_file_path = ( + log_dir + / f"openhcs_zmq_server_port_{self.port}_{int(time.time() * 1000000)}.log" + ) + cmd = [ + sys.executable, + "-m", + "openhcs.runtime.zmq_execution_server_launcher", + "--port", + str(self.port), + ] if self.persistent: cmd.append("--persistent") cmd.extend(["--log-file-path", str(log_file_path)]) cmd.extend(["--transport-mode", self.transport_mode.value]) + # Pass the current process's logging level to the server + # Get the root logger's effective level + root_logger = logging.getLogger() + current_log_level = root_logger.getEffectiveLevel() + log_level_name = logging.getLevelName(current_log_level) + + # Log what we're passing to help debug + logger = logging.getLogger(__name__) + logger.debug(f"Spawning ZMQ server with log level: {log_level_name} (numeric: {current_log_level})") + + cmd.extend(["--log-level", log_level_name]) + env = os.environ.copy() site_packages = ( Path(sys.executable).parent.parent @@ -111,12 +203,18 @@ def _spawn_server_process(self): / "site-packages" ) nvidia_lib_pattern = str(site_packages / "nvidia" / "*" / "lib") - venv_nvidia_libs = [p for p in glob.glob(nvidia_lib_pattern) if os.path.isdir(p)] + venv_nvidia_libs = [ + p for p in glob.glob(nvidia_lib_pattern) if os.path.isdir(p) + ] if venv_nvidia_libs: existing_ld_path = env.get("LD_LIBRARY_PATH", "") nvidia_paths = ":".join(venv_nvidia_libs) - env["LD_LIBRARY_PATH"] = f"{nvidia_paths}:{existing_ld_path}" if existing_ld_path else nvidia_paths + env["LD_LIBRARY_PATH"] = ( + f"{nvidia_paths}:{existing_ld_path}" + if existing_ld_path + else nvidia_paths + ) return subprocess.Popen( cmd, @@ -135,6 +233,3 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.disconnect() - - -ZMQExecutionClient = OpenHCSExecutionClient diff --git a/openhcs/runtime/zmq_execution_server.py b/openhcs/runtime/zmq_execution_server.py index 3249b4e5b..a617b300d 100644 --- a/openhcs/runtime/zmq_execution_server.py +++ b/openhcs/runtime/zmq_execution_server.py @@ -1,25 +1,67 @@ """OpenHCS execution server built on zmqruntime ExecutionServer.""" + from __future__ import annotations import logging import time +import threading +import hashlib +import json from typing import Any from zmqruntime.execution import ExecutionServer -from zmqruntime.messages import ExecuteRequest, ExecutionStatus, MessageFields +from zmqruntime.messages import ( + ExecuteRequest, + ExecutionStatus, + MessageFields, + ResponseType, + StatusRequest, +) from zmqruntime.transport import coerce_transport_mode from openhcs.runtime.zmq_config import OPENHCS_ZMQ_CONFIG +from openhcs.core.progress import ProgressPhase, ProgressStatus, ProgressEvent + logger = logging.getLogger(__name__) -class OpenHCSExecutionServer(ExecutionServer): +def _emit_zmq_progress(enqueue_fn, **kwargs) -> None: + """Helper to emit progress to ZMQ queue using new schema. + + All fields are explicit on ProgressEvent - just pass kwargs directly. + """ + import time + import os + + # Create event - all fields are explicit, just pass kwargs directly + event = ProgressEvent(timestamp=time.time(), pid=os.getpid(), **kwargs) + enqueue_fn(event.to_dict()) + + +class _ImmediateProgressQueue: + """Queue adapter that forwards progress updates immediately to ZMQ.""" + + def __init__(self, server: "ZMQExecutionServer"): + self._server = server + + def put(self, progress_update: dict) -> None: + self._server._enqueue_progress(progress_update) + self._server._flush_progress_only() + + +class ZMQExecutionServer(ExecutionServer): """OpenHCS-specific execution server.""" _server_type = "execution" - def __init__(self, port: int | None = None, host: str = "*", log_file_path: str | None = None, transport_mode=None): + def __init__( + self, + port: int | None = None, + host: str = "*", + log_file_path: str | None = None, + transport_mode=None, + ): super().__init__( port=port or OPENHCS_ZMQ_CONFIG.default_port, host=host, @@ -27,6 +69,293 @@ def __init__(self, port: int | None = None, host: str = "*", log_file_path: str transport_mode=coerce_transport_mode(transport_mode), config=OPENHCS_ZMQ_CONFIG, ) + self._compile_status: str | None = None + self._compile_message: str | None = None + self._compile_status_expires_at: float | None = None + self._worker_assignments_by_execution: dict[str, dict[str, list[str]]] = {} + self._compiled_artifacts: dict[str, dict[str, Any]] = {} + self._compiled_artifact_ttl_seconds: float = 30.0 * 60.0 + + @staticmethod + def _extract_compiled_axis_ids(compiled_contexts: dict[str, Any]) -> list[str]: + """Extract unique axis ids from compiled context keys. + + Context keys may be either plain axis ids (e.g. "A01") or sequential keys + (e.g. "A01__combo_0"). Worker ownership must always be at axis granularity. + """ + axis_ids: list[str] = [] + seen: set[str] = set() + for context_key in compiled_contexts.keys(): + axis_id = ( + context_key.split("__combo_", 1)[0] + if "__combo_" in context_key + else context_key + ) + if axis_id not in seen: + seen.add(axis_id) + axis_ids.append(axis_id) + return sorted(axis_ids) + + @staticmethod + def _validate_worker_claim( + worker_slot: str, + owned_wells: list[str], + assignments: dict[str, list[str]], + ) -> None: + if worker_slot not in assignments: + raise ValueError( + f"Unknown worker_slot '{worker_slot}'. Expected one of: {list(assignments.keys())}" + ) + expected = assignments[worker_slot] + if sorted(owned_wells) != sorted(expected): + raise ValueError( + f"Invalid worker claim for {worker_slot}: expected={expected}, got={owned_wells}" + ) + + def _flush_progress_only(self) -> None: + """Flush only progress messages to ZMQ, without processing control messages. + + This is called during synchronous execution when we can't receive from + the control socket (would block or fail with NOBLOCK). + """ + if not self.data_socket: + return + import json + import queue + + logger = logging.getLogger(__name__) + count = 0 + while True: + try: + progress_update = self.progress_queue.get_nowait() + except queue.Empty: + if count > 0: + logger.info(f"Flushed {count} progress update(s) to ZMQ") + break + logger.info( + f"Flushing to ZMQ: step_name={progress_update.get('step_name')!r}, " + f"axis={progress_update.get('axis_id')!r}, plate_id={progress_update.get('plate_id')!r}, " + f"percent={progress_update.get('percent')!r}, total_wells={progress_update.get('total_wells')!r}" + ) + json_str = json.dumps(progress_update) + logger.info(f"Full JSON being sent: {json_str[:300]}") + self.data_socket.send_string(json_str) + count += 1 + + def _set_compile_status( + self, status: str, message: str | None = None, ttl_seconds: float = 4.0 + ) -> None: + self._compile_status = status + self._compile_message = message + self._compile_status_expires_at = time.time() + ttl_seconds + + def _get_compile_status(self) -> tuple[str | None, str | None]: + if self._compile_status_expires_at is None: + return None, None + if time.time() > self._compile_status_expires_at: + self._compile_status = None + self._compile_message = None + self._compile_status_expires_at = None + return None, None + return self._compile_status, self._compile_message + + @staticmethod + def _build_request_signature( + plate_id: str, + pipeline_code: str, + config_params: dict | None, + config_code: str | None, + pipeline_config_code: str | None, + ) -> str: + payload = { + MessageFields.PLATE_ID: plate_id, + MessageFields.PIPELINE_CODE: pipeline_code, + MessageFields.CONFIG_PARAMS: config_params, + MessageFields.CONFIG_CODE: config_code, + MessageFields.PIPELINE_CONFIG_CODE: pipeline_config_code, + } + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + def _cleanup_compiled_artifacts(self) -> None: + now = time.time() + expired_ids = [ + artifact_id + for artifact_id, artifact in self._compiled_artifacts.items() + if now - artifact["created_at"] > self._compiled_artifact_ttl_seconds + ] + for artifact_id in expired_ids: + del self._compiled_artifacts[artifact_id] + if expired_ids: + logger.info("Cleaned up %d expired compile artifact(s)", len(expired_ids)) + + def _create_pong_response(self): + self._cleanup_compiled_artifacts() + response = super()._create_pong_response() + + # Add OpenHCS-specific compile status + compile_status, compile_message = self._get_compile_status() + if compile_status is not None: + response[MessageFields.COMPILE_STATUS] = compile_status + if compile_message is not None: + response[MessageFields.COMPILE_MESSAGE] = compile_message + + queued = [ + (execution_id, record) + for execution_id, record in self.active_executions.items() + if record.status == ExecutionStatus.QUEUED.value + ] + response["queued_executions"] = [ + { + MessageFields.EXECUTION_ID: execution_id, + MessageFields.PLATE_ID: str(record.plate_id), + "queue_position": index + 1, + } + for index, (execution_id, record) in enumerate(queued) + ] + + return response + + def _enqueue_progress(self, progress_update: dict) -> None: + # DEBUG: Log what's being enqueued + if "total_wells" in progress_update: + logger.info( + f"_enqueue_progress: total_wells={progress_update.get('total_wells')}, keys={list(progress_update.keys())}, step_name={progress_update.get('step_name')}" + ) + self.progress_queue.put(progress_update) + + def _forward_worker_progress(self, worker_queue) -> None: + import logging + + logger = logging.getLogger(__name__) + while True: + progress_update = worker_queue.get() + if progress_update is None: + logger.info("Progress forwarder received None, exiting") + break + execution_id = progress_update.get("execution_id") + if not execution_id: + raise ValueError( + f"Worker progress missing execution_id: {progress_update}" + ) + assignments = self._worker_assignments_by_execution.get(execution_id) + if assignments is None: + raise ValueError( + f"Missing worker assignments for execution_id={execution_id}" + ) + + # Pipeline-level INIT events (e.g. viewer launch) bypass worker + # claim validation — they carry no worker_slot / owned_wells. + phase = progress_update.get("phase") + axis_id = progress_update.get("axis_id", "") + if phase == "init" and not axis_id: + self.progress_queue.put(progress_update) + continue + + worker_slot = progress_update.get("worker_slot") + owned_wells = progress_update.get("owned_wells") + if not worker_slot or owned_wells is None: + raise ValueError( + f"Worker progress missing claim fields: worker_slot={worker_slot}, owned_wells={owned_wells}" + ) + self._validate_worker_claim(worker_slot, owned_wells, assignments) + # Attach topology metadata to every worker progress event so the UI + # cannot lose worker/well ownership due first-message ordering. + progress_update = dict(progress_update) + progress_update["worker_assignments"] = assignments + progress_update["total_wells"] = sorted( + { + axis_id + for assigned_axes in assignments.values() + for axis_id in assigned_axes + } + ) + logger.info( + f"Forwarding progress: pid={progress_update.get('pid')}, axis={progress_update.get('axis_id')}, step_name={progress_update.get('step_name')}, worker_slot={worker_slot}" + ) + self.progress_queue.put(progress_update) + + def _get_worker_info(self): + """Return raw worker info (no enrichment). + + Worker axis_id comes from progress tracker, not from ping responses. + Ping is for process tracking (CPU, memory), not application state. + """ + return super()._get_worker_info() + + def _attach_results_summary_extras( + self, execution_id: str, record, execution_payload: dict | None = None + ) -> None: + if record.status != ExecutionStatus.COMPLETE.value: + return + + summary = record.results_summary + if not isinstance(summary, dict): + summary = {} + record.results_summary = summary + + output_plate_root = record.get_extra("output_plate_root") + auto_add_output_plate = record.get_extra("auto_add_output_plate") + if output_plate_root: + summary["output_plate_root"] = str(output_plate_root) + if auto_add_output_plate is not None: + summary["auto_add_output_plate_to_plate_manager"] = bool( + auto_add_output_plate + ) + + if isinstance(execution_payload, dict): + execution_payload[MessageFields.RESULTS_SUMMARY] = summary + + logger.info( + "[%s] Attached results_summary extras: output_plate_root=%s auto_add=%s", + execution_id, + summary.get("output_plate_root"), + summary.get("auto_add_output_plate_to_plate_manager"), + ) + + def _run_execution(self, execution_id, request, record): + """Run an execution and enrich results_summary with output plate path. + + The base zmqruntime ExecutionServer only populates well_count/wells in + results_summary. OpenHCS needs the final output plate root (computed by + path planning during compilation) so the UI can optionally auto-add it + as a new orchestrator in Plate Manager. + """ + super()._run_execution(execution_id, request, record) + + try: + self._attach_results_summary_extras( + execution_id=execution_id, record=record + ) + except Exception as e: + logger.warning( + "[%s] Failed to attach output_plate_root to results_summary: %s", + execution_id, + e, + ) + + def _handle_status(self, msg): + response = super()._handle_status(msg) + if response.get(MessageFields.STATUS) != ResponseType.OK.value: + return response + + execution_id = StatusRequest.from_dict(msg).execution_id + if not execution_id: + return response + + record = self.active_executions[execution_id] + if record.status != ExecutionStatus.COMPLETE.value: + return response + + execution_payload = response.get(MessageFields.EXECUTION) + self._attach_results_summary_extras( + execution_id=execution_id, + record=record, + execution_payload=execution_payload + if isinstance(execution_payload, dict) + else None, + ) + return response def execute_task(self, execution_id: str, request: ExecuteRequest) -> Any: return self._execute_pipeline( @@ -37,6 +366,8 @@ def execute_task(self, execution_id: str, request: ExecuteRequest) -> Any: request.config_code, request.pipeline_config_code, request.client_address, + request.compile_only, + request.compile_artifact_id, ) def _execute_pipeline( @@ -48,6 +379,8 @@ def _execute_pipeline( config_code, pipeline_config_code, client_address=None, + compile_only: bool = False, + compile_artifact_id: str | None = None, ): from openhcs.constants import AllComponents, VariableComponents, GroupBy from openhcs.core.config import GlobalPipelineConfig, PipelineConfig @@ -55,23 +388,63 @@ def _execute_pipeline( logger.info("[%s] Starting plate %s", execution_id, plate_id) import openhcs.processing.func_registry as func_registry_module - logger.info("[%s] Registry initialized status BEFORE check: %s", execution_id, func_registry_module._registry_initialized) + + logger.info( + "[%s] Registry initialized status BEFORE check: %s", + execution_id, + func_registry_module._registry_initialized, + ) with func_registry_module._registry_lock: if not func_registry_module._registry_initialized: logger.info("[%s] Initializing registry...", execution_id) func_registry_module._auto_initialize_registry() - logger.info("[%s] Registry initialized status AFTER init: %s", execution_id, func_registry_module._registry_initialized) + logger.info( + "[%s] Registry initialized status AFTER init: %s", + execution_id, + func_registry_module._registry_initialized, + ) else: logger.info("[%s] Registry already initialized, skipping", execution_id) + self._cleanup_compiled_artifacts() + + if compile_only and compile_artifact_id: + raise ValueError("compile_only and compile_artifact_id cannot both be set") + + request_signature = self._build_request_signature( + plate_id=plate_id, + pipeline_code=pipeline_code, + config_params=config_params, + config_code=config_code, + pipeline_config_code=pipeline_config_code, + ) + pipeline_sha = hashlib.sha256(pipeline_code.encode("utf-8")).hexdigest()[:12] + namespace = {} exec(pipeline_code, namespace) if not (pipeline_steps := namespace.get("pipeline_steps")): raise ValueError("Code must define 'pipeline_steps'") + logger.info( + "[%s] Request received: plate=%s compile_only=%s artifact_id=%s step_count=%d pipeline_sha=%s request_sig=%s", + execution_id, + plate_id, + bool(compile_only), + compile_artifact_id, + len(pipeline_steps), + pipeline_sha, + request_signature[:12], + ) if config_code: - is_empty = "GlobalPipelineConfig(\n\n)" in config_code or "GlobalPipelineConfig()" in config_code - global_config = GlobalPipelineConfig() if is_empty else (exec(config_code, ns := {}) or ns.get("config")) + is_empty = ( + "GlobalPipelineConfig(\n\n)" in config_code + or "GlobalPipelineConfig()" in config_code + ) + global_config = ( + GlobalPipelineConfig() + if is_empty + else (exec(config_code, ns := {}) or ns.get("config")) + ) if not global_config: raise ValueError("config_code must define 'config'") pipeline_config = ( @@ -82,18 +455,28 @@ def _execute_pipeline( if pipeline_config_code and not pipeline_config: raise ValueError("pipeline_config_code must define 'config'") elif config_params: - global_config, pipeline_config = self._build_config_from_params(config_params) + global_config, pipeline_config = self._build_config_from_params( + config_params + ) else: raise ValueError("Either config_params or config_code required") - return self._execute_with_orchestrator( - execution_id, - plate_id, - pipeline_steps, - global_config, - pipeline_config, - config_params, - ) + try: + return self._execute_with_orchestrator( + execution_id, + plate_id, + pipeline_steps, + global_config, + pipeline_config, + config_params, + compile_only=compile_only, + compile_artifact_id=compile_artifact_id, + request_signature=request_signature, + ) + except Exception as e: + if compile_only: + self._set_compile_status("compiled failed", str(e)) + raise def _build_config_from_params(self, p): from openhcs.core.config import ( @@ -108,23 +491,46 @@ def _build_config_from_params(self, p): return ( GlobalPipelineConfig( num_workers=p.get("num_workers", 4), - path_planning_config=PathPlanningConfig(output_dir_suffix=p.get("output_dir_suffix", "_output")), + path_planning_config=PathPlanningConfig( + output_dir_suffix=p.get("output_dir_suffix", "_output") + ), vfs_config=VFSConfig( - materialization_backend=MaterializationBackend(p.get("materialization_backend", "disk")) + materialization_backend=MaterializationBackend( + p.get("materialization_backend", "disk") + ) + ), + step_well_filter_config=StepWellFilterConfig( + well_filter=p.get("well_filter") ), - step_well_filter_config=StepWellFilterConfig(well_filter=p.get("well_filter")), use_threading=p.get("use_threading", False), ), PipelineConfig(), ) - def _execute_with_orchestrator(self, execution_id, plate_id, pipeline_steps, global_config, pipeline_config, config_params): + def _execute_with_orchestrator( + self, + execution_id, + plate_id, + pipeline_steps, + global_config, + pipeline_config, + config_params, + compile_only: bool = False, + compile_artifact_id: str | None = None, + request_signature: str | None = None, + ): from pathlib import Path import multiprocessing from openhcs.config_framework.lazy_factory import ensure_global_config_context + from openhcs.core.config import GlobalPipelineConfig from openhcs.core.orchestrator.gpu_scheduler import setup_global_gpu_registry from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator - from openhcs.constants import AllComponents, VariableComponents, GroupBy, MULTIPROCESSING_AXIS + from openhcs.constants import ( + AllComponents, + VariableComponents, + GroupBy, + MULTIPROCESSING_AXIS, + ) from polystore.base import reset_memory_backend, storage_registry try: @@ -140,10 +546,17 @@ def _execute_with_orchestrator(self, execution_id, plate_id, pipeline_steps, glo cleanup_all_gpu_frameworks() except Exception as cleanup_error: - logger.warning("[%s] Failed to trigger GPU cleanup: %s", execution_id, cleanup_error) + logger.warning( + "[%s] Failed to trigger GPU cleanup: %s", execution_id, cleanup_error + ) + + if not isinstance(global_config, GlobalPipelineConfig): + raise TypeError( + f"Expected GlobalPipelineConfig, got {type(global_config).__name__}" + ) setup_global_gpu_registry(global_config=global_config) - ensure_global_config_context(type(global_config), global_config) + ensure_global_config_context(GlobalPipelineConfig, global_config) plate_path_str = str(plate_id) is_omero_plate_id = False @@ -161,7 +574,7 @@ def _execute_with_orchestrator(self, execution_id, plate_id, pipeline_steps, glo from openhcs.runtime.omero_instance_manager import OMEROInstanceManager from openhcs.microscopes import omero # noqa: F401 - Import OMERO parsers to register them from polystore.omero_local import OMEROLocalBackend - + omero_manager = OMEROInstanceManager() if not omero_manager.connect(timeout=60): raise RuntimeError("OMERO server not available") @@ -174,49 +587,374 @@ def _execute_with_orchestrator(self, execution_id, plate_id, pipeline_steps, glo if not plate_path_str.startswith("/omero/"): plate_path_str = f"/omero/plate_{plate_path_str}" - orchestrator = PipelineOrchestrator( - plate_path=Path(plate_path_str), - pipeline_config=pipeline_config, - progress_callback=None, - ) - orchestrator.initialize() - self.active_executions[execution_id]["orchestrator"] = orchestrator - - if self.active_executions[execution_id][MessageFields.STATUS] == ExecutionStatus.CANCELLED.value: - logger.info("[%s] Execution cancelled after initialization, aborting", execution_id) - raise RuntimeError("Execution cancelled by user") + progress_context = { + MessageFields.EXECUTION_ID: execution_id, + MessageFields.PLATE_ID: plate_id, + MessageFields.AXIS_ID: "", + } + worker_progress_queue = None + progress_forwarder = None + compiled_contexts: dict[str, Any] | None = None - wells = config_params.get("well_filter") if config_params else orchestrator.get_component_keys(MULTIPROCESSING_AXIS) - compilation = orchestrator.compile_pipelines(pipeline_definition=pipeline_steps, well_filter=wells) - - if self.active_executions[execution_id][MessageFields.STATUS] == ExecutionStatus.CANCELLED.value: - logger.info("[%s] Execution cancelled after compilation, aborting", execution_id) - raise RuntimeError("Execution cancelled by user") + try: + if compile_artifact_id is None: + _emit_zmq_progress( + self._enqueue_progress, + execution_id=execution_id, + plate_id=plate_id, + axis_id="", + step_name="pipeline", + total=len(pipeline_steps), + phase=ProgressPhase.COMPILE, + status=ProgressStatus.STARTED, + completed=0, + percent=0.0, + ) + orchestrator = PipelineOrchestrator( + plate_path=Path(plate_path_str), + pipeline_config=pipeline_config, + progress_callback=None, + ) + orchestrator.execution_id = execution_id + orchestrator.initialize() + self.active_executions[execution_id].set_extra("orchestrator", orchestrator) + + if ( + self.active_executions[execution_id].status + == ExecutionStatus.CANCELLED.value + ): + logger.info( + "[%s] Execution cancelled after initialization, aborting", + execution_id, + ) + raise RuntimeError("Execution cancelled by user") + + if config_params and config_params.get("well_filter"): + wells = list(config_params["well_filter"]) + else: + wells = orchestrator.get_component_keys(MULTIPROCESSING_AXIS) + + # Initialize compiled_pipeline_definition - will be set by either artifact reuse or fresh compilation + compiled_pipeline_definition = None + + if compile_artifact_id is None: + planned_metadata = {"total_wells": sorted(wells)} + step_names = [step.name for step in pipeline_steps] + planned_metadata["step_names"] = step_names + _emit_zmq_progress( + self._enqueue_progress, + execution_id=execution_id, + plate_id=plate_id, + axis_id="", + step_name="", + phase=ProgressPhase.INIT, + status=ProgressStatus.STARTED, + percent=0.0, + completed=0, + total=1, + **planned_metadata, + ) + + if compile_artifact_id is not None: + self._cleanup_compiled_artifacts() + artifact = self._compiled_artifacts.pop(compile_artifact_id, None) + if artifact is None: + raise ValueError( + f"Missing compile artifact '{compile_artifact_id}'. " + "Re-run compilation before execution." + ) + if request_signature is None: + raise ValueError( + "Missing request signature for artifact validation" + ) + if artifact["request_signature"] != request_signature: + logger.error( + "[%s] Compile artifact signature mismatch: artifact_id=%s artifact_sig=%s request_sig=%s", + execution_id, + compile_artifact_id, + str(artifact["request_signature"])[:12], + request_signature[:12], + ) + raise ValueError( + f"Compile artifact '{compile_artifact_id}' does not match execution request" + ) + if artifact[MessageFields.PLATE_ID] != str(plate_id): + raise ValueError( + f"Compile artifact '{compile_artifact_id}' is for plate " + f"{artifact[MessageFields.PLATE_ID]}, not {plate_id}" + ) + + compiled_contexts = artifact["compiled_contexts"] + if compiled_contexts is None: + raise ValueError("Compile artifact missing compiled_contexts") + compiled_pipeline_definition = artifact.get( + "compiled_pipeline_definition" + ) # Get the stripped pipeline_definition from artifact + worker_assignments = artifact["worker_assignments"] + self._worker_assignments_by_execution[execution_id] = worker_assignments + compiled_axis_ids = self._extract_compiled_axis_ids(compiled_contexts) + + # Emit filtered metadata for this execution id. + step_names = [step.name for step in pipeline_steps] + _emit_zmq_progress( + self._enqueue_progress, + execution_id=execution_id, + plate_id=plate_id, + axis_id="", + step_name="", + phase=ProgressPhase.INIT, + status=ProgressStatus.STARTED, + percent=0.0, + completed=0, + total=1, + total_wells=compiled_axis_ids, + worker_assignments=worker_assignments, + step_names=step_names, + ) + + logger.info( + "[%s] Reused compile artifact %s for plate %s (sig=%s)", + execution_id, + compile_artifact_id, + plate_id, + request_signature[:12] if request_signature else "missing", + ) + + output_plate_root = artifact.get("output_plate_root") + if output_plate_root: + self.active_executions[execution_id].set_extra( + "output_plate_root", + str(output_plate_root), + ) + if artifact.get("auto_add_output_plate") is not None: + self.active_executions[execution_id].set_extra( + "auto_add_output_plate", bool(artifact["auto_add_output_plate"]) + ) + else: + # Compilation runs in THIS process (queue worker thread), not a + # separate worker process. Use an immediate adapter so compile + # events are forwarded to ZMQ as soon as they are emitted. + from openhcs.core.progress import set_progress_queue - log_dir = Path.home() / ".local" / "share" / "openhcs" / "logs" - log_dir.mkdir(parents=True, exist_ok=True) + set_progress_queue(_ImmediateProgressQueue(self)) + try: + compilation = orchestrator.compile_pipelines( + pipeline_definition=pipeline_steps, + well_filter=wells, + is_zmq_execution=True, + ) + finally: + set_progress_queue(None) + + if ( + not isinstance(compilation, dict) + or "compiled_contexts" not in compilation + ): + raise ValueError("Compilation did not return compiled_contexts") + compiled_contexts = compilation["compiled_contexts"] + # CRITICAL: Use the returned pipeline_definition, not the original pipeline_steps + # The compiler modifies pipeline_definition in-place (converts functions to FunctionReference) + # and returns the modified version + compiled_pipeline_definition = compilation.get( + "pipeline_definition", pipeline_steps + ) + + # DEBUG: Check if they're the same object + logger.info( + f"🔍 ZMQ: pipeline_steps is compiled_pipeline_definition? {pipeline_steps is compiled_pipeline_definition}" + ) + logger.info( + f"🔍 ZMQ: pipeline_steps[0].func = {type(getattr(pipeline_steps[0], 'func', None)).__name__ if getattr(pipeline_steps[0], 'func', None) else 'None'}" + ) + logger.info( + f"🔍 ZMQ: compiled_pipeline_definition[0].func = {type(getattr(compiled_pipeline_definition[0], 'func', None)).__name__ if getattr(compiled_pipeline_definition[0], 'func', None) else 'None'}" + ) + if not compiled_contexts: + raise ValueError("Compilation produced no compiled contexts") + + # Get worker_assignments from compiler result (uses PipelineConfig's num_workers) + worker_assignments = compilation["worker_assignments"] + self._worker_assignments_by_execution[execution_id] = worker_assignments + compiled_axis_ids = self._extract_compiled_axis_ids(compiled_contexts) + + # Emit filtered metadata for this execution id. + _emit_zmq_progress( + self._enqueue_progress, + execution_id=execution_id, + plate_id=plate_id, + axis_id="", + step_name="", + phase=ProgressPhase.INIT, + status=ProgressStatus.STARTED, + percent=0.0, + completed=0, + total=1, + total_wells=compiled_axis_ids, + worker_assignments=worker_assignments, + ) + + _emit_zmq_progress( + self._enqueue_progress, + execution_id=execution_id, + plate_id=plate_id, + axis_id="", + step_name="pipeline", + total=len(pipeline_steps), + phase=ProgressPhase.COMPILE, + status=ProgressStatus.SUCCESS, + completed=1, + percent=100.0, + total_wells=compiled_axis_ids, + worker_assignments=worker_assignments, + ) + self._flush_progress_only() + + for axis_id in compiled_axis_ids: + _emit_zmq_progress( + self._enqueue_progress, + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name="compilation", + phase=ProgressPhase.COMPILE, + status=ProgressStatus.SUCCESS, + completed=1, + total=1, + percent=100.0, + ) + self._flush_progress_only() + + first_context = next(iter(compiled_contexts.values())) + output_plate_root = first_context.output_plate_root + if output_plate_root: + self.active_executions[execution_id].set_extra( + "output_plate_root", + str(output_plate_root), + ) + self.active_executions[execution_id].set_extra( + "auto_add_output_plate", + bool(first_context.auto_add_output_plate_to_plate_manager), + ) + logger.info( + "[%s] Captured auto_add_output_plate=%s output_plate_root=%s", + execution_id, + bool(first_context.auto_add_output_plate_to_plate_manager), + output_plate_root, + ) + + if ( + self.active_executions[execution_id].status + == ExecutionStatus.CANCELLED.value + ): + logger.info( + "[%s] Execution cancelled after compilation, aborting", + execution_id, + ) + raise RuntimeError("Execution cancelled by user") + + if compile_only: + if request_signature is None: + raise ValueError( + "Missing request signature for compile artifact storage" + ) + self._compiled_artifacts[execution_id] = { + "created_at": time.time(), + "request_signature": request_signature, + MessageFields.PLATE_ID: str(plate_id), + "compiled_contexts": compiled_contexts, + "compiled_pipeline_definition": compiled_pipeline_definition, # Store the stripped pipeline_definition + "worker_assignments": worker_assignments, + "output_plate_root": self.active_executions[execution_id].get_extra( + "output_plate_root" + ), + "auto_add_output_plate": self.active_executions[ + execution_id + ].get_extra("auto_add_output_plate"), + } + logger.info( + "[%s] Compilation-only request completed and artifact stored (artifact_id=%s sig=%s)", + execution_id, + execution_id, + request_signature[:12], + ) + self._set_compile_status("compiled success") + return compiled_contexts + + log_dir = Path.home() / ".local" / "share" / "openhcs" / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + + worker_progress_queue = multiprocessing.get_context("spawn").Queue() + progress_forwarder = threading.Thread( + target=self._forward_worker_progress, + args=(worker_progress_queue,), + daemon=True, + ) + progress_forwarder.start() + + if ( + self.active_executions[execution_id].status + == ExecutionStatus.CANCELLED.value + ): + logger.info( + "[%s] Execution cancelled before starting workers, aborting", + execution_id, + ) + raise RuntimeError("Execution cancelled by user") + + # DEBUG: Check steps right before execution + logger.info( + f"🔍 PRE-EXEC: compiled_pipeline_definition is None? {compiled_pipeline_definition is None}" + ) + if compiled_pipeline_definition is not None: + logger.info( + f"🔍 PRE-EXEC: compiled_pipeline_definition[0].func = {type(getattr(compiled_pipeline_definition[0], 'func', None)).__name__ if getattr(compiled_pipeline_definition[0], 'func', None) else 'None'}" + ) + logger.info( + f"🔍 PRE-EXEC: pipeline_steps[0].func = {type(getattr(pipeline_steps[0], 'func', None)).__name__ if getattr(pipeline_steps[0], 'func', None) else 'None'}" + ) - if self.active_executions[execution_id][MessageFields.STATUS] == ExecutionStatus.CANCELLED.value: - logger.info("[%s] Execution cancelled before starting workers, aborting", execution_id) - raise RuntimeError("Execution cancelled by user") + # Use compiled pipeline_definition if available (from fresh compilation), + # otherwise use original pipeline_steps (from artifact reuse) + # NOTE: For artifact reuse, we don't have the compiled pipeline_definition, + # so we use the original pipeline_steps (functions will be resolved from context) + steps_to_execute = ( + compiled_pipeline_definition + if compiled_pipeline_definition is not None + else pipeline_steps + ) - return orchestrator.execute_compiled_plate( - pipeline_definition=pipeline_steps, - compiled_contexts=compilation["compiled_contexts"], - log_file_base=str(log_dir / f"zmq_worker_exec_{execution_id}"), - ) + # DEBUG: Log what we're passing to execution + logger.info( + f"🚀 ZMQ SERVER: Passing {len(steps_to_execute)} steps to execution" + ) + for i, step in enumerate(steps_to_execute): + func_attr = getattr(step, "func", None) + func_type = type(func_attr).__name__ if func_attr else "None" + logger.info(f"🚀 ZMQ SERVER: step[{i}].func = {func_type}") + + return orchestrator.execute_compiled_plate( + pipeline_definition=steps_to_execute, + compiled_contexts=compiled_contexts, + log_file_base=str(log_dir / f"zmq_worker_exec_{execution_id}"), + progress_queue=worker_progress_queue, + progress_context=progress_context, + worker_assignments=worker_assignments, + ) + finally: + if worker_progress_queue is not None: + worker_progress_queue.put(None) + if progress_forwarder is not None: + progress_forwarder.join() + self._worker_assignments_by_execution.pop(execution_id, None) def _kill_worker_processes(self) -> int: """OpenHCS-specific worker cleanup (graceful cancellation + kill).""" - for eid, r in self.active_executions.items(): - if "orchestrator" in r: + for eid, record in self.active_executions.items(): + orchestrator = record.get_extra("orchestrator") + if orchestrator is not None: try: logger.info("[%s] Requesting graceful cancellation...", eid) - r["orchestrator"].cancel_execution() + orchestrator.cancel_execution() except Exception as e: logger.warning("[%s] Graceful cancellation failed: %s", eid, e) return super()._kill_worker_processes() - - -# Backwards-compatible alias -ZMQExecutionServer = OpenHCSExecutionServer diff --git a/openhcs/runtime/zmq_execution_server_launcher.py b/openhcs/runtime/zmq_execution_server_launcher.py index 97b43020b..a808c1507 100644 --- a/openhcs/runtime/zmq_execution_server_launcher.py +++ b/openhcs/runtime/zmq_execution_server_launcher.py @@ -12,13 +12,8 @@ from zmqruntime.transport import get_default_transport_mode from zmqruntime.runner import serve_forever -from openhcs.runtime.zmq_execution_server import OpenHCSExecutionServer +from openhcs.runtime.zmq_execution_server import ZMQExecutionServer -# Set up logging -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", -) logger = logging.getLogger(__name__) @@ -46,14 +41,39 @@ def main(): choices=["ipc", "tcp"], help=f"Transport mode (default: {default_mode_str} for this platform)", ) + parser.add_argument( + "--log-level", + type=str, + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + help="Logging level (default: INFO)", + ) args = parser.parse_args() + # Configure logging with the specified level + # CRITICAL: Must force reconfigure root logger - basicConfig() does nothing if already configured + log_level = getattr(logging, args.log_level.upper()) + root_logger = logging.getLogger() + root_logger.setLevel(log_level) + + # Also set level on all existing handlers (in case they were configured with a different level) + for handler in root_logger.handlers: + handler.setLevel(log_level) + + # If no handlers exist, add a basic console handler + if not root_logger.handlers: + console_handler = logging.StreamHandler() + console_handler.setLevel(log_level) + console_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) + root_logger.addHandler(console_handler) + transport_mode = TransportMode.IPC if args.transport_mode == "ipc" else TransportMode.TCP logger.info("=" * 60) logger.info("ZMQ Execution Server") logger.info("=" * 60) + logger.info("Log level: %s (from --log-level=%s)", logging.getLevelName(log_level), args.log_level) logger.info("Port: %s (control: %s)", args.port, args.port + 1000) logger.info("Host: %s", args.host) logger.info("Transport mode: %s", transport_mode.value) @@ -62,7 +82,7 @@ def main(): logger.info("Log file: %s", args.log_file_path) logger.info("=" * 60) - server = OpenHCSExecutionServer( + server = ZMQExecutionServer( port=args.port, host=args.host, log_file_path=args.log_file_path, diff --git a/openhcs/serialization/pycodify_formatters.py b/openhcs/serialization/pycodify_formatters.py index dfbef3666..cc4899e7d 100644 --- a/openhcs/serialization/pycodify_formatters.py +++ b/openhcs/serialization/pycodify_formatters.py @@ -4,6 +4,7 @@ from typing import Any, Dict, Tuple from openhcs.core.steps.function_step import FunctionStep +from pyqt_reactive.services.pattern_data_manager import SCOPE_TOKEN_KEY from pycodify import FormatContext, SourceFormatter, SourceFragment, to_source @@ -53,6 +54,13 @@ def _is_pattern_item(value: Any) -> bool: return callable(value) or _is_pattern_tuple(value) +def _strip_internal_pattern_metadata(args: Dict[str, Any]) -> Dict[str, Any]: + """Remove UI-only metadata keys from function-pattern kwargs.""" + if not isinstance(args, dict): + return {} + return {k: v for k, v in args.items() if k != SCOPE_TOKEN_KEY} + + class FunctionPatternTupleFormatter(SourceFormatter): priority = 85 @@ -61,6 +69,7 @@ def can_format(self, value: Any) -> bool: def format(self, value: Tuple[Any, Dict[str, Any]], context: FormatContext) -> SourceFragment: func, args = value + args = _strip_internal_pattern_metadata(args) if not args and context.clean_mode: return to_source(func, context) diff --git a/openhcs/tests/test_pipeline.py b/openhcs/tests/test_pipeline.py index c98c672fc..480de44db 100644 --- a/openhcs/tests/test_pipeline.py +++ b/openhcs/tests/test_pipeline.py @@ -83,23 +83,13 @@ # Step 8: Cell Counting step_8 = FunctionStep( - func={ '1': (count_cells_single_channel, { + func= (count_cells_single_channel, { 'min_cell_area': 40, 'max_cell_area': 200, 'enable_preprocessing': False, 'detection_method': DetectionMethod.WATERSHED, - 'dtype_conversion': DtypeConversion.UINT8, 'return_segmentation_mask': True }), - '2': (count_cells_single_channel, { - 'min_cell_area': 40, - 'max_cell_area': 200, - 'enable_preprocessing': False, - 'detection_method': DetectionMethod.WATERSHED, - 'dtype_conversion': DtypeConversion.UINT8, - 'return_segmentation_mask': True - }) - }, name="Cell Counting", napari_streaming_config=LazyNapariStreamingConfig(variable_size_handling=NapariVariableSizeHandling.PAD_TO_MAX), ) diff --git a/openhcs/textual_tui/widgets/plate_manager.py b/openhcs/textual_tui/widgets/plate_manager.py index f08aa349f..98de88110 100644 --- a/openhcs/textual_tui/widgets/plate_manager.py +++ b/openhcs/textual_tui/widgets/plate_manager.py @@ -39,37 +39,37 @@ from openhcs.constants.constants import Backend, VariableComponents, OrchestratorState from openhcs.textual_tui.services.file_browser_service import SelectionMode from openhcs.textual_tui.services.window_service import WindowService -from openhcs.core.path_cache import get_cached_browser_path, PathCacheKey, get_path_cache +from openhcs.core.path_cache import ( + get_cached_browser_path, + PathCacheKey, + get_path_cache, +) from openhcs.introspection import SignatureAnalyzer logger = logging.getLogger(__name__) # Note: Using subprocess approach instead of multiprocessing to avoid TUI FD conflicts + def get_orchestrator_status_symbol(orchestrator: PipelineOrchestrator) -> str: """Get UI symbol for orchestrator state - simple mapping without over-engineering.""" if orchestrator is None: return "?" # No orchestrator (newly added plate) state_to_symbol = { - OrchestratorState.CREATED: "?", # Created but not initialized - OrchestratorState.READY: "-", # Initialized, ready for compilation - OrchestratorState.COMPILED: "o", # Compiled, ready for execution - OrchestratorState.EXECUTING: "!", # Execution in progress - OrchestratorState.COMPLETED: "C", # Execution completed successfully - OrchestratorState.INIT_FAILED: "I", # Initialization failed + OrchestratorState.CREATED: "?", # Created but not initialized + OrchestratorState.READY: "-", # Initialized, ready for compilation + OrchestratorState.COMPILED: "o", # Compiled, ready for execution + OrchestratorState.EXECUTING: "!", # Execution in progress + OrchestratorState.COMPLETED: "C", # Execution completed successfully + OrchestratorState.INIT_FAILED: "I", # Initialization failed OrchestratorState.COMPILE_FAILED: "P", # Compilation failed (P for Pipeline) - OrchestratorState.EXEC_FAILED: "X", # Execution failed + OrchestratorState.EXEC_FAILED: "X", # Execution failed } return state_to_symbol.get(orchestrator.state, "?") - - - - - class PlateManagerWidget(ButtonListWidget): """ Plate management widget using Textual reactive state. @@ -80,17 +80,21 @@ class PlateManagerWidget(ButtonListWidget): orchestrators = reactive({}) plate_configs = reactive({}) orchestrator_state_version = reactive(0) # Increment to trigger UI refresh - + def __init__(self, filemanager: FileManager, global_config: GlobalPipelineConfig): button_configs = [ ButtonConfig("Add", "add_plate"), ButtonConfig("Del", "del_plate", disabled=True), - ButtonConfig("Edit", "edit_config", disabled=True), # Unified edit button for config editing + ButtonConfig( + "Edit", "edit_config", disabled=True + ), # Unified edit button for config editing ButtonConfig("Init", "init_plate", disabled=True), ButtonConfig("Compile", "compile_plate", disabled=True), ButtonConfig("Run", "run_plate", disabled=True), ButtonConfig("Code", "code_plate", disabled=True), # Generate Python code - ButtonConfig("Save", "save_python_script", disabled=True), # Save Python script + ButtonConfig( + "Save", "save_python_script", disabled=True + ), # Save Python script # ButtonConfig("Export", "export_ome_zarr", disabled=True), # Export to OME-ZARR - HIDDEN FROM UI ] super().__init__( @@ -99,13 +103,13 @@ def __init__(self, filemanager: FileManager, global_config: GlobalPipelineConfig container_id="plate_list", on_button_pressed=self._handle_button_press, on_selection_changed=self._handle_selection_change, - on_item_moved=self._handle_item_moved + on_item_moved=self._handle_item_moved, ) self.filemanager = filemanager self.global_config = global_config self.plate_compiled_data = {} self.on_plate_selected: Optional[Callable[[str], None]] = None - self.pipeline_editor: Optional['PipelineEditorWidget'] = None + self.pipeline_editor: Optional["PipelineEditorWidget"] = None # Initialize window service to avoid circular imports self.window_service = None # Will be set in on_mount @@ -115,22 +119,23 @@ def __init__(self, filemanager: FileManager, global_config: GlobalPipelineConfig self.zmq_client = None # ZMQ execution client (when using ZMQ mode) self.current_execution_id = None # Track current execution ID for cancellation self.log_file_path: Optional[str] = None # Single source of truth - self.log_file_position: int = 0 # Track position in log file for incremental reading + self.log_file_position: int = ( + 0 # Track position in log file for incremental reading + ) # Async monitoring using Textual's interval system self.monitoring_interval = None self.monitoring_active = False # --- - - logger.debug("PlateManagerWidget initialized") - - - + logger.debug("PlateManagerWidget initialized") def on_unmount(self) -> None: - logger.debug("Unmounting PlateManagerWidget, ensuring worker process is terminated.") + logger.debug( + "Unmounting PlateManagerWidget, ensuring worker process is terminated." + ) # Schedule async stop execution since on_unmount is sync import asyncio + if self.current_process and self.current_process.poll() is None: # Create a task to stop execution asynchronously asyncio.create_task(self.action_stop_execution()) @@ -138,22 +143,22 @@ def on_unmount(self) -> None: def format_item_for_display(self, plate: Dict) -> Tuple[str, str]: # Get status from orchestrator instead of magic string - plate_path = plate.get('path', '') + plate_path = plate.get("path", "") orchestrator = self.orchestrators.get(plate_path) status_symbol = get_orchestrator_status_symbol(orchestrator) status_symbols = { - "?": "➕", # Created (not initialized) - "-": "✅", # Ready (initialized) - "o": "⚡", # Compiled - "!": "🔄", # Executing - "C": "🏁", # Completed - "I": "🚫", # Init failed - "P": "💥", # Compile failed (Pipeline) - "X": "❌" # Execution failed + "?": "➕", # Created (not initialized) + "-": "✅", # Ready (initialized) + "o": "⚡", # Compiled + "!": "🔄", # Executing + "C": "🏁", # Completed + "I": "🚫", # Init failed + "P": "💥", # Compile failed (Pipeline) + "X": "❌", # Execution failed } status_icon = status_symbols.get(status_symbol, "❓") - plate_name = plate.get('name', 'Unknown') + plate_name = plate.get("name", "Unknown") display_text = f"{status_icon} {plate_name} - {plate_path}" return display_text, plate_path @@ -188,7 +193,7 @@ def _handle_item_moved(self, from_index: int, to_index: int) -> None: plate = current_plates.pop(from_index) current_plates.insert(to_index, plate) self.items = current_plates - plate_name = plate['name'] + plate_name = plate["name"] direction = "up" if to_index < from_index else "down" self.app.current_status = f"Moved plate '{plate_name}' {direction}" @@ -198,19 +203,23 @@ def on_mount(self) -> None: self.call_later(self._delayed_update_display) self.call_later(self._update_button_states) - + def watch_items(self, items: List[Dict]) -> None: """Automatically update UI when items changes (follows ButtonListWidget pattern).""" # DEBUG: Log when items list changes to track the source of the reset - stack_trace = ''.join(traceback.format_stack()[-3:-1]) # Get last 2 stack frames - logger.debug(f"🔍 ITEMS CHANGED: {len(items)} plates. Call stack:\n{stack_trace}") + stack_trace = "".join( + traceback.format_stack()[-3:-1] + ) # Get last 2 stack frames + logger.debug( + f"🔍 ITEMS CHANGED: {len(items)} plates. Call stack:\n{stack_trace}" + ) # CRITICAL: Call parent's watch_items to update the SelectionList super().watch_items(items) logger.debug(f"Plates updated: {len(items)} plates") self._update_button_states() - + def watch_highlighted_item(self, plate_path: str) -> None: self.selected_plate = plate_path logger.debug(f"Highlighted plate: {plate_path}") @@ -236,7 +245,9 @@ def watch_orchestrator_state_version(self, version: int) -> None: # Also notify PipelineEditor if connected if self.pipeline_editor: - logger.debug(f"PlateManager: Notifying PipelineEditor of orchestrator state change (version {version})") + logger.debug( + f"PlateManager: Notifying PipelineEditor of orchestrator state change (version {version})" + ) self.pipeline_editor._update_button_states() def get_selection_state(self) -> tuple[List[Dict], str]: @@ -249,16 +260,20 @@ def get_selection_state(self) -> tuple[List[Dict], str]: selection_list = self.query_one(f"#{self.list_id}") multi_selected_values = selection_list.selected if multi_selected_values: - selected_items = [p for p in self.items if p.get('path') in multi_selected_values] + selected_items = [ + p for p in self.items if p.get("path") in multi_selected_values + ] return selected_items, "checkbox" elif self.selected_plate: - selected_items = [p for p in self.items if p.get('path') == self.selected_plate] + selected_items = [ + p for p in self.items if p.get("path") == self.selected_plate + ] return selected_items, "cursor" else: return [], "empty" except Exception as e: # DOM CORRUPTION DETECTED - This is a critical error - stack_trace = ''.join(traceback.format_stack()[-3:-1]) + stack_trace = "".join(traceback.format_stack()[-3:-1]) logger.error(f"🚨 DOM CORRUPTION: Failed to get selection state: {e}") logger.error(f"🚨 DOM CORRUPTION: Call stack:\n{stack_trace}") logger.error(f"🚨 DOM CORRUPTION: Widget mounted: {self.is_mounted}") @@ -274,14 +289,22 @@ def get_selection_state(self) -> tuple[List[Dict], str]: logger.error(f"🚨 DOM CORRUPTION: Could not diagnose widgets: {diag_e}") if self.selected_plate: - selected_items = [p for p in self.items if p.get('path') == self.selected_plate] + selected_items = [ + p for p in self.items if p.get("path") == self.selected_plate + ] return selected_items, "cursor" return [], "empty" - def get_operation_description(self, selected_items: List[Dict], selection_mode: str, operation: str) -> str: + def get_operation_description( + self, selected_items: List[Dict], selection_mode: str, operation: str + ) -> str: count = len(selected_items) - if count == 0: return f"No items for {operation}" - if count == 1: return f"{operation.title()} item: {selected_items[0].get('name', 'Unknown')}" + if count == 0: + return f"No items for {operation}" + if count == 1: + return ( + f"{operation.title()} item: {selected_items[0].get('name', 'Unknown')}" + ) return f"{operation.title()} {count} items" def _delayed_update_display(self) -> None: @@ -308,7 +331,11 @@ def _update_button_states(self) -> None: selected_items, _ = self.get_selection_state() has_selected_items = bool(selected_items) - can_run = has_selection and any(p['path'] in self.plate_compiled_data for p in self.items if p.get('path') == self.selected_plate) + can_run = has_selection and any( + p["path"] in self.plate_compiled_data + for p in self.items + if p.get("path") == self.selected_plate + ) # Try to get run button - if it doesn't exist, widget is not fully mounted try: @@ -324,47 +351,64 @@ def _update_button_states(self) -> None: return self.query_one("#add_plate").disabled = is_running - self.query_one("#del_plate").disabled = not self.items or not has_selected_items or is_running + self.query_one("#del_plate").disabled = ( + not self.items or not has_selected_items or is_running + ) # Edit button (config editing) enabled when 1+ orchestrators selected and initialized selected_items, _ = self.get_selection_state() edit_enabled = ( - len(selected_items) > 0 and - all(self._is_orchestrator_initialized(item['path']) for item in selected_items) and - not is_running + len(selected_items) > 0 + and all( + self._is_orchestrator_initialized(item["path"]) + for item in selected_items + ) + and not is_running ) self.query_one("#edit_config").disabled = not edit_enabled # Init button - enabled when plates are selected, can be initialized, and not running init_enabled = ( - len(selected_items) > 0 and - any(self._can_orchestrator_be_initialized(item['path']) for item in selected_items) and - not is_running + len(selected_items) > 0 + and any( + self._can_orchestrator_be_initialized(item["path"]) + for item in selected_items + ) + and not is_running ) self.query_one("#init_plate").disabled = not init_enabled # Compile button - enabled when plates are selected, initialized, and not running selected_items, _ = self.get_selection_state() compile_enabled = ( - len(selected_items) > 0 and - all(self._is_orchestrator_initialized(item['path']) for item in selected_items) and - not is_running + len(selected_items) > 0 + and all( + self._is_orchestrator_initialized(item["path"]) + for item in selected_items + ) + and not is_running ) self.query_one("#compile_plate").disabled = not compile_enabled # Code button - enabled when plates are selected, initialized, and not running code_enabled = ( - len(selected_items) > 0 and - all(self._is_orchestrator_initialized(item['path']) for item in selected_items) and - not is_running + len(selected_items) > 0 + and all( + self._is_orchestrator_initialized(item["path"]) + for item in selected_items + ) + and not is_running ) self.query_one("#code_plate").disabled = not code_enabled # Save Python script button - enabled when plates are selected, initialized, and not running save_enabled = ( - len(selected_items) > 0 and - all(self._is_orchestrator_initialized(item['path']) for item in selected_items) and - not is_running + len(selected_items) > 0 + and all( + self._is_orchestrator_initialized(item["path"]) + for item in selected_items + ) + and not is_running ) self.query_one("#save_python_script").disabled = not save_enabled @@ -396,7 +440,7 @@ def _has_pipelines(self, plates: List[Dict]) -> bool: return False for plate in plates: - pipeline = self.pipeline_editor.get_pipeline_for_plate(plate['path']) + pipeline = self.pipeline_editor.get_pipeline_for_plate(plate["path"]) if not pipeline: return False return True @@ -411,18 +455,27 @@ def _is_orchestrator_initialized(self, plate_path: str) -> bool: orchestrator = self.orchestrators.get(plate_path) if orchestrator is None: return False - return orchestrator.state in [OrchestratorState.READY, OrchestratorState.COMPILED, - OrchestratorState.COMPLETED, OrchestratorState.COMPILE_FAILED, - OrchestratorState.EXEC_FAILED] + return orchestrator.state in [ + OrchestratorState.READY, + OrchestratorState.COMPILED, + OrchestratorState.COMPLETED, + OrchestratorState.COMPILE_FAILED, + OrchestratorState.EXEC_FAILED, + ] def _can_orchestrator_be_initialized(self, plate_path: str) -> bool: """Check if orchestrator can be initialized (doesn't exist or is in a re-initializable state).""" orchestrator = self.orchestrators.get(plate_path) if orchestrator is None: return True # No orchestrator exists, can be initialized - return orchestrator.state in [OrchestratorState.CREATED, OrchestratorState.INIT_FAILED] + return orchestrator.state in [ + OrchestratorState.CREATED, + OrchestratorState.INIT_FAILED, + ] - def _notify_pipeline_editor_status_change(self, plate_path: str, new_status: str) -> None: + def _notify_pipeline_editor_status_change( + self, plate_path: str, new_status: str + ) -> None: """Notify pipeline editor when plate status changes (enables Add button immediately).""" if self.pipeline_editor and self.pipeline_editor.current_plate == plate_path: # Update pipeline editor's status and trigger button state update @@ -446,7 +499,9 @@ def _get_current_pipeline_definition(self, plate_path: str = None) -> List: # No conversion needed - pipeline_steps are already FunctionStep objects with memory type decorators return pipeline_steps - def get_operation_description(self, selected_items: List[Dict], selection_mode: str, operation: str) -> str: + def get_operation_description( + self, selected_items: List[Dict], selection_mode: str, operation: str + ) -> str: """Generate human-readable description of what will be operated on.""" count = len(selected_items) if selection_mode == "empty": @@ -455,14 +510,16 @@ def get_operation_description(self, selected_items: List[Dict], selection_mode: return f"{operation.title()} ALL {count} items" elif selection_mode == "checkbox": if count == 1: - item_name = selected_items[0].get('name', 'Unknown') + item_name = selected_items[0].get("name", "Unknown") return f"{operation.title()} selected item: {item_name}" else: return f"{operation.title()} {count} selected items" else: return f"{operation.title()} {count} items" - def _reset_execution_state(self, status_message: str, force_fail_executing: bool = True): + def _reset_execution_state( + self, status_message: str, force_fail_executing: bool = True + ): if self.current_process: if self.current_process.poll() is None: # Still running logger.warning("Forcefully terminating subprocess during reset.") @@ -492,7 +549,7 @@ def _reset_execution_state(self, status_message: str, force_fail_executing: bool # Update button states - but only if widget is properly mounted try: - if self.is_mounted and hasattr(self, 'query_one'): + if self.is_mounted and hasattr(self, "query_one"): self._update_button_states() except Exception as e: logger.error(f"Failed to update button states during reset: {e}") @@ -503,12 +560,14 @@ async def action_run_plate(self) -> None: # Clear logs from singleton toolong window before starting new run try: from openhcs.textual_tui.windows.toolong_window import clear_toolong_logs + logger.info("Clearing logs from singleton toolong window before new run") clear_toolong_logs(self.app) logger.info("Toolong logs cleared") except Exception as e: logger.error(f"Failed to clear toolong logs: {e}") import traceback + logger.error(traceback.format_exc()) selected_items, _ = self.get_selection_state() @@ -516,9 +575,15 @@ async def action_run_plate(self) -> None: self.app.show_error("No plates selected to run.") return - ready_items = [item for item in selected_items if item.get('path') in self.plate_compiled_data] + ready_items = [ + item + for item in selected_items + if item.get("path") in self.plate_compiled_data + ] if not ready_items: - self.app.show_error("Selected plates are not compiled. Please compile first.") + self.app.show_error( + "Selected plates are not compiled. Please compile first." + ) return await self._run_plates_zmq(ready_items) @@ -532,10 +597,10 @@ def _start_log_monitoring(self) -> None: try: # Extract base path from log file path (remove .log extension) log_path = Path(self.log_file_path) - base_log_path = str(log_path.with_suffix('')) + base_log_path = str(log_path.with_suffix("")) # Notify status bar to start log monitoring - if hasattr(self.app, 'status_bar') and self.app.status_bar: + if hasattr(self.app, "status_bar") and self.app.status_bar: self.app.status_bar.start_log_monitoring(base_log_path) logger.debug(f"Started reactive log monitoring for: {base_log_path}") else: @@ -548,7 +613,7 @@ def _stop_log_monitoring(self) -> None: """Stop reactive log monitoring.""" try: # Notify status bar to stop log monitoring - if hasattr(self.app, 'status_bar') and self.app.status_bar: + if hasattr(self.app, "status_bar") and self.app.status_bar: self.app.status_bar.stop_log_monitoring() logger.debug("Stopped reactive log monitoring") except Exception as e: @@ -563,13 +628,11 @@ def _get_current_log_position(self) -> int: except Exception: return 0 - - def _stop_file_watcher(self) -> None: """Stop file system watcher without blocking.""" if not self.file_observer: return - + try: # Just stop and abandon - don't wait for anything self.file_observer.stop() @@ -580,8 +643,6 @@ def _stop_file_watcher(self) -> None: self.file_observer = None self.file_watcher = None - - async def _start_monitoring(self) -> None: """Start async monitoring using Textual's interval system.""" # Stop any existing monitoring @@ -595,7 +656,7 @@ async def _start_monitoring(self) -> None: self.monitoring_interval = self.set_interval( 10.0, # Check every 10 seconds self._check_process_status_async, - pause=False + pause=False, ) logger.debug("Started async process monitoring") @@ -639,11 +700,15 @@ async def _handle_process_completion(self) -> None: if self.log_file_path and Path(self.log_file_path).exists(): try: # Read log file directly to check for success markers - with open(self.log_file_path, 'r') as f: + with open(self.log_file_path, "r") as f: log_content = f.read() # Look for success markers in the log - has_execution_success = "🔥 SUBPROCESS: EXECUTION SUCCESS:" in log_content - has_all_completed = "All plates completed successfully" in log_content + has_execution_success = ( + "🔥 SUBPROCESS: EXECUTION SUCCESS:" in log_content + ) + has_all_completed = ( + "All plates completed successfully" in log_content + ) if has_execution_success and has_all_completed: success = True @@ -668,7 +733,9 @@ async def _handle_process_completion(self) -> None: orchestrator._state = OrchestratorState.COMPLETED # Reset execution state (this will trigger UI refresh internally) - self._reset_execution_state("Execution completed successfully.", force_fail_executing=False) + self._reset_execution_state( + "Execution completed successfully.", force_fail_executing=False + ) else: # Failure - update orchestrators to failed for plate_path, orchestrator in self.orchestrators.items(): @@ -692,7 +759,7 @@ def _read_log_content(): if not Path(self.log_file_path).exists(): return None, self.log_file_position - with open(self.log_file_path, 'r') as f: + with open(self.log_file_path, "r") as f: # Seek to where we left off f.seek(self.log_file_position) new_content = f.read() @@ -701,7 +768,9 @@ def _read_log_content(): return new_content, new_position - new_content, new_position = await asyncio.get_event_loop().run_in_executor(None, _read_log_content) + new_content, new_position = await asyncio.get_event_loop().run_in_executor( + None, _read_log_content + ) self.log_file_position = new_position if new_content is None: @@ -710,7 +779,7 @@ def _read_log_content(): if new_content and new_content.strip(): # Get the last non-empty line from new content - lines = new_content.strip().split('\n') + lines = new_content.strip().split("\n") non_empty_lines = [line.strip() for line in lines if line.strip()] if non_empty_lines: @@ -728,21 +797,19 @@ def _read_log_content(): except Exception as e: self.app.current_status = f"🔥 LOG READER ERROR: {e}" - - async def _run_plates_zmq(self, ready_items) -> None: """Run plates using ZMQ execution client (recommended).""" try: from openhcs.runtime.zmq_execution_client import ZMQExecutionClient - plate_paths_to_run = [item['path'] for item in ready_items] + plate_paths_to_run = [item["path"] for item in ready_items] logger.info(f"Starting ZMQ execution for {len(plate_paths_to_run)} plates") # Create ZMQ client (non-persistent mode for UI-managed execution) self.zmq_client = ZMQExecutionClient( port=7777, persistent=False, # UI manages lifecycle - progress_callback=self._on_zmq_progress + progress_callback=self._on_zmq_progress, ) # Connect to server (will spawn if needed) @@ -750,6 +817,7 @@ def _connect(): return self.zmq_client.connect(timeout=15) import asyncio + loop = asyncio.get_event_loop() connected = await loop.run_in_executor(None, _connect) @@ -760,7 +828,7 @@ def _connect(): # Update orchestrator states to show running state for plate in ready_items: - plate_path = plate['path'] + plate_path = plate["path"] if plate_path in self.orchestrators: self.orchestrators[plate_path]._state = OrchestratorState.EXECUTING @@ -775,6 +843,7 @@ def _connect(): # Get effective config for this plate effective_config = self.app.global_config from openhcs.core.config import PipelineConfig + pipeline_config = PipelineConfig() logger.info(f"Executing plate: {plate_path}") @@ -785,28 +854,38 @@ def _execute(): plate_id=str(plate_path), pipeline_steps=definition_pipeline, global_config=effective_config, - pipeline_config=pipeline_config + pipeline_config=pipeline_config, ) response = await loop.run_in_executor(None, _execute) # Track execution ID for cancellation - if response.get('execution_id'): - self.current_execution_id = response['execution_id'] + if response.get("execution_id"): + self.current_execution_id = response["execution_id"] - logger.info(f"Plate {plate_path} execution response: {response.get('status')}") + logger.info( + f"Plate {plate_path} execution response: {response.get('status')}" + ) - if response.get('status') != 'complete': - error_msg = response.get('message', 'Unknown error') - traceback_str = response.get('traceback', '') + if response.get("status") != "complete": + error_msg = response.get("message", "Unknown error") + traceback_str = response.get("traceback", "") # Log full traceback if available if traceback_str: - logger.error(f"Plate {plate_path} execution failed:\n{traceback_str}") - self.app.show_error(f"Execution failed for {plate_path}:\n\n{traceback_str}") + logger.error( + f"Plate {plate_path} execution failed:\n{traceback_str}" + ) + self.app.show_error( + f"Execution failed for {plate_path}:\n\n{traceback_str}" + ) else: - logger.error(f"Plate {plate_path} execution failed: {error_msg}") - self.app.show_error(f"Execution failed for {plate_path}: {error_msg}") + logger.error( + f"Plate {plate_path} execution failed: {error_msg}" + ) + self.app.show_error( + f"Execution failed for {plate_path}: {error_msg}" + ) # Execution complete self.current_execution_id = None @@ -814,9 +893,11 @@ def _execute(): # Update orchestrator states for plate in ready_items: - plate_path = plate['path'] + plate_path = plate["path"] if plate_path in self.orchestrators: - self.orchestrators[plate_path]._state = OrchestratorState.EXEC_COMPLETE + self.orchestrators[ + plate_path + ]._state = OrchestratorState.EXEC_COMPLETE self._trigger_ui_refresh() self._update_button_states() @@ -835,7 +916,7 @@ def _disconnect(): self._reset_execution_state("ZMQ execution failed") # Cleanup ZMQ client - if hasattr(self, 'zmq_client') and self.zmq_client: + if hasattr(self, "zmq_client") and self.zmq_client: try: self.zmq_client.disconnect() except: @@ -844,18 +925,15 @@ def _disconnect(): def _on_zmq_progress(self, message): """Handle progress updates from ZMQ execution server.""" - try: - well_id = message.get('well_id', 'unknown') - step = message.get('step', 'unknown') - status = message.get('status', 'unknown') + plate_id = message["plate_id"] + axis_id = message["axis_id"] + step_name = message["step_name"] + phase = message["phase"] + percent = message["percent"] - # Update status in TUI - progress_text = f"[{well_id}] {step}: {status}" - self.app.current_status = progress_text - logger.debug(f"Progress: {progress_text}") - - except Exception as e: - logger.warning(f"Failed to handle progress update: {e}") + progress_text = f"{plate_id} [{axis_id}] {step_name} {percent:.0f}% {phase}" + self.app.current_status = progress_text + logger.debug(f"Progress: {progress_text}") async def action_stop_execution(self) -> None: logger.info("🛑 Stop button pressed.") @@ -870,6 +948,7 @@ async def action_stop_execution(self) -> None: logger.info("🛑 Requesting graceful cancellation via ZMQ...") import asyncio + loop = asyncio.get_event_loop() # Cancel specific execution if we have an ID @@ -877,12 +956,16 @@ async def action_stop_execution(self) -> None: logger.info(f"🛑 Cancelling execution {self.current_execution_id}") def _cancel(): - return self.zmq_client.cancel_execution(self.current_execution_id) + return self.zmq_client.cancel_execution( + self.current_execution_id + ) response = await loop.run_in_executor(None, _cancel) - if response.get('status') == 'ok': - logger.info("🛑 Cancellation request accepted, waiting for graceful shutdown...") + if response.get("status") == "ok": + logger.info( + "🛑 Cancellation request accepted, waiting for graceful shutdown..." + ) self.app.current_status = "Cancellation requested, waiting..." # Wait for graceful cancellation with timeout @@ -892,23 +975,37 @@ def _cancel(): while (asyncio.get_event_loop().time() - start_time) < timeout: # Check if execution is still running def _check_status(): - return self.zmq_client.get_status(self.current_execution_id) + return self.zmq_client.get_status( + self.current_execution_id + ) - status_response = await loop.run_in_executor(None, _check_status) + status_response = await loop.run_in_executor( + None, _check_status + ) - if status_response.get('status') == 'error': + if status_response.get("status") == "error": # Execution no longer exists (completed or cancelled) - logger.info("🛑 Execution completed/cancelled gracefully") + logger.info( + "🛑 Execution completed/cancelled gracefully" + ) break await asyncio.sleep(0.5) else: # Timeout reached - execution still running - logger.warning("🛑 Graceful cancellation timeout - execution may still be running") - self.app.current_status = "Cancellation timeout - execution may still be running" + logger.warning( + "🛑 Graceful cancellation timeout - execution may still be running" + ) + self.app.current_status = ( + "Cancellation timeout - execution may still be running" + ) else: - logger.warning(f"🛑 Cancellation failed: {response.get('message')}") - self.app.current_status = f"Cancellation failed: {response.get('message')}" + logger.warning( + f"🛑 Cancellation failed: {response.get('message')}" + ) + self.app.current_status = ( + f"Cancellation failed: {response.get('message')}" + ) # Disconnect client def _disconnect(): @@ -924,11 +1021,15 @@ def _disconnect(): logger.error(f"🛑 Error cancelling ZMQ execution: {e}") self.app.show_error(f"Failed to cancel execution: {e}") - elif self.current_process and self.current_process.poll() is None: # Still running subprocess + elif ( + self.current_process and self.current_process.poll() is None + ): # Still running subprocess try: # Kill the entire process group, not just the parent process # The subprocess creates its own process group, so we need to kill that group - logger.info(f"🛑 Killing process group for PID {self.current_process.pid}...") + logger.info( + f"🛑 Killing process group for PID {self.current_process.pid}..." + ) # Get the process group ID (should be same as PID since subprocess calls os.setpgrp()) process_group_id = self.current_process.pid @@ -944,17 +1045,19 @@ def _disconnect(): os.killpg(process_group_id, signal.SIGKILL) logger.info(f"🛑 Force killed process group {process_group_id}") except ProcessLookupError: - logger.info(f"🛑 Process group {process_group_id} already terminated") + logger.info( + f"🛑 Process group {process_group_id} already terminated" + ) except Exception as e: - logger.warning(f"🛑 Error killing process group: {e}, falling back to single process kill") + logger.warning( + f"🛑 Error killing process group: {e}, falling back to single process kill" + ) # Fallback to killing just the main process self.current_process.kill() self._reset_execution_state("Execution terminated by user.") - - async def action_add_plate(self) -> None: """Handle Add Plate button.""" await self._open_plate_directory_browser() @@ -968,13 +1071,19 @@ async def action_export_ome_zarr(self) -> None: # Get the orchestrator for the selected plate orchestrator = self.orchestrators.get(self.selected_plate) if not orchestrator: - self.app.show_error("Not Initialized", "Please initialize the plate before exporting.") + self.app.show_error( + "Not Initialized", "Please initialize the plate before exporting." + ) return # Open file browser for export location def handle_export_result(selected_paths): if selected_paths: - export_path = Path(selected_paths[0]) if isinstance(selected_paths, list) else Path(selected_paths) + export_path = ( + Path(selected_paths[0]) + if isinstance(selected_paths, list) + else Path(selected_paths) + ) self._start_ome_zarr_export(orchestrator, export_path) await self.window_service.open_file_browser( @@ -986,26 +1095,32 @@ def handle_export_result(selected_paths): selection_mode=SelectionMode.DIRECTORIES_ONLY, cache_key=PathCacheKey.GENERAL, on_result_callback=handle_export_result, - caller_id="plate_manager_export" + caller_id="plate_manager_export", ) def _start_ome_zarr_export(self, orchestrator, export_path: Path): """Start OME-ZARR export process.""" + async def run_export(): try: self.app.current_status = f"Exporting to OME-ZARR: {export_path}" # Create export-specific config with ZARR materialization from openhcs.core.config import GlobalPipelineConfig - from openhcs.config_framework.global_config import get_current_global_config + from openhcs.config_framework.global_config import ( + get_current_global_config, + ) + export_config = get_current_global_config(GlobalPipelineConfig) export_vfs_config = VFSConfig( intermediate_backend=export_config.vfs_config.intermediate_backend, - materialization_backend=MaterializationBackend.ZARR + materialization_backend=MaterializationBackend.ZARR, ) # Update orchestrator config for export - export_global_config = dataclasses.replace(export_config, vfs=export_vfs_config) + export_global_config = dataclasses.replace( + export_config, vfs=export_vfs_config + ) # Create zarr backend with OME-ZARR enabled zarr_backend = ZarrStorageBackend(ome_zarr_metadata=True) @@ -1025,7 +1140,7 @@ async def run_export(): # Extract well from filename well_match = None # Try ImageXpress pattern: A01_s001_w1_z001.tif - match = re.search(r'([A-Z]\d{2})_', img_path.name) + match = re.search(r"([A-Z]\d{2})_", img_path.name) if match: well_id = match.group(1) wells_data[well_id].append(img_path) @@ -1041,21 +1156,33 @@ async def run_export(): images.append(np.array(img)) # Create output paths for OME-ZARR structure - output_paths = [export_store_path / f"{well_id}_{i:03d}.tif" - for i in range(len(images))] + output_paths = [ + export_store_path / f"{well_id}_{i:03d}.tif" + for i in range(len(images)) + ] # Save to OME-ZARR format - zarr_backend.save_batch(images, output_paths, chunk_name=well_id) + zarr_backend.save_batch( + images, output_paths, chunk_name=well_id + ) - self.app.current_status = f"✅ OME-ZARR export completed: {export_store_path}" + self.app.current_status = ( + f"✅ OME-ZARR export completed: {export_store_path}" + ) else: - self.app.show_error("No Data", "No processed images found in workspace.") + self.app.show_error( + "No Data", "No processed images found in workspace." + ) else: - self.app.show_error("No Workspace", "Plate workspace not found. Run pipeline first.") + self.app.show_error( + "No Workspace", "Plate workspace not found. Run pipeline first." + ) except Exception as e: logger.error(f"OME-ZARR export failed: {e}", exc_info=True) - self.app.show_error("Export Failed", f"OME-ZARR export failed: {str(e)}") + self.app.show_error( + "Export Failed", f"OME-ZARR export failed: {str(e)}" + ) # Run export in background asyncio.create_task(run_export()) @@ -1066,7 +1193,9 @@ async def _open_plate_directory_browser(self): """Open textual-window file browser for plate directory selection.""" # Get cached path for better UX - remembers last used directory path_cache = get_path_cache() - initial_path = path_cache.get_initial_path(PathCacheKey.PLATE_IMPORT, Path.home()) + initial_path = path_cache.get_initial_path( + PathCacheKey.PLATE_IMPORT, Path.home() + ) # Open textual-window file browser for directory selection await self.window_service.open_file_browser( @@ -1079,12 +1208,14 @@ async def _open_plate_directory_browser(self): cache_key=PathCacheKey.PLATE_IMPORT, on_result_callback=self._add_plate_callback, caller_id="plate_manager", - enable_multi_selection=True + enable_multi_selection=True, ) def _add_plate_callback(self, selected_paths) -> None: """Handle directory selection from file browser.""" - logger.debug(f"_add_plate_callback called with: {selected_paths} (type: {type(selected_paths)})") + logger.debug( + f"_add_plate_callback called with: {selected_paths} (type: {type(selected_paths)})" + ) if selected_paths is None or selected_paths is False: self.app.current_status = "Plate selection cancelled" @@ -1105,15 +1236,15 @@ def _add_plate_callback(self, selected_paths) -> None: selected_path = Path(str(selected_path)) # Check if plate already exists - if any(plate['path'] == str(selected_path) for plate in current_plates): + if any(plate["path"] == str(selected_path) for plate in current_plates): continue # Add the plate to the list plate_name = selected_path.name plate_path = str(selected_path) plate_entry = { - 'name': plate_name, - 'path': plate_path, + "name": plate_name, + "path": plate_path, # No status field - state comes from orchestrator } @@ -1123,7 +1254,11 @@ def _add_plate_callback(self, selected_paths) -> None: # Cache the parent directory for next time (save user navigation time) if selected_paths: # Use parent of first selected path as the cached directory - first_path = selected_paths[0] if isinstance(selected_paths[0], Path) else Path(selected_paths[0]) + first_path = ( + selected_paths[0] + if isinstance(selected_paths[0], Path) + else Path(selected_paths[0]) + ) parent_dir = first_path.parent get_path_cache().set_cached_path(PathCacheKey.PLATE_IMPORT, parent_dir) @@ -1134,7 +1269,9 @@ def _add_plate_callback(self, selected_paths) -> None: if len(added_plates) == 1: self.app.current_status = f"Added plate: {added_plates[0]}" else: - self.app.current_status = f"Added {len(added_plates)} plates: {', '.join(added_plates)}" + self.app.current_status = ( + f"Added {len(added_plates)} plates: {', '.join(added_plates)}" + ) else: self.app.current_status = "No new plates added (duplicates skipped)" @@ -1143,9 +1280,9 @@ def action_delete_plate(self) -> None: if not selected_items: self.app.show_error("No plate selected to delete.") return - - paths_to_delete = {p['path'] for p in selected_items} - self.items = [p for p in self.items if p['path'] not in paths_to_delete] + + paths_to_delete = {p["path"] for p in selected_items} + self.items = [p for p in self.items if p["path"] not in paths_to_delete] # Clean up orchestrators for deleted plates for path in paths_to_delete: @@ -1157,8 +1294,6 @@ def action_delete_plate(self) -> None: self.app.current_status = f"Deleted {len(paths_to_delete)} plate(s)" - - async def action_edit_config(self) -> None: """ Handle Edit button - create per-orchestrator PipelineConfig instances. @@ -1173,8 +1308,9 @@ async def action_edit_config(self) -> None: return selected_orchestrators = [ - self.orchestrators[item['path']] for item in selected_items - if item['path'] in self.orchestrators + self.orchestrators[item["path"]] + for item in selected_items + if item["path"] in self.orchestrators ] if not selected_orchestrators: @@ -1185,9 +1321,13 @@ async def action_edit_config(self) -> None: representative_orchestrator = selected_orchestrators[0] # Use orchestrator's existing config if it exists, otherwise use global config as source - source_config = representative_orchestrator.pipeline_config or self.global_config + source_config = ( + representative_orchestrator.pipeline_config or self.global_config + ) - current_plate_config = create_dataclass_for_editing(PipelineConfig, source_config) + current_plate_config = create_dataclass_for_editing( + PipelineConfig, source_config + ) def handle_config_save(new_config: PipelineConfig) -> None: """Apply per-orchestrator configuration without global side effects.""" @@ -1195,15 +1335,14 @@ def handle_config_save(new_config: PipelineConfig) -> None: # Direct synchronous call - no async needed orchestrator.apply_pipeline_config(new_config) - count = len(selected_orchestrators) - self.app.current_status = f"Per-orchestrator configuration applied to {count} orchestrator(s)" + self.app.current_status = ( + f"Per-orchestrator configuration applied to {count} orchestrator(s)" + ) # Open configuration window using PipelineConfig (not GlobalPipelineConfig) await self.window_service.open_config_window( - PipelineConfig, - current_plate_config, - on_save_callback=handle_config_save + PipelineConfig, current_plate_config, on_save_callback=handle_config_save ) async def action_edit_global_config(self) -> None: @@ -1212,7 +1351,7 @@ async def action_edit_global_config(self) -> None: This maintains the existing global configuration workflow but uses lazy loading. """ - + from openhcs.core.config import PipelineConfig from openhcs.config_framework.lazy_factory import create_dataclass_for_editing @@ -1220,7 +1359,9 @@ async def action_edit_global_config(self) -> None: current_global_config = self.app.global_config or GlobalPipelineConfig() # Create lazy PipelineConfig for editing with proper thread-local context - current_lazy_config = create_dataclass_for_editing(PipelineConfig, current_global_config, preserve_values=True) + current_lazy_config = create_dataclass_for_editing( + PipelineConfig, current_global_config, preserve_values=True + ) def handle_global_config_save(new_config: PipelineConfig) -> None: """Apply global configuration to all orchestrators.""" @@ -1233,18 +1374,20 @@ def handle_global_config_save(new_config: PipelineConfig) -> None: for orchestrator in self.orchestrators.values(): asyncio.create_task(orchestrator.apply_new_global_config(global_config)) - self.app.current_status = "Global configuration applied to all orchestrators" + self.app.current_status = ( + "Global configuration applied to all orchestrators" + ) # PipelineConfig already imported from openhcs.core.config await self.window_service.open_config_window( PipelineConfig, current_lazy_config, - on_save_callback=handle_global_config_save + on_save_callback=handle_global_config_save, ) - - - def _analyze_orchestrator_configs(self, orchestrators: List['PipelineOrchestrator']) -> Dict[str, Dict[str, Any]]: + def _analyze_orchestrator_configs( + self, orchestrators: List["PipelineOrchestrator"] + ) -> Dict[str, Dict[str, Any]]: """Analyze configs across multiple orchestrators to detect same/different values. Args: @@ -1289,13 +1432,13 @@ def _analyze_orchestrator_configs(self, orchestrators: List['PipelineOrchestrato config_analysis[field_name] = { "type": "same", "value": values[0], - "default": default_value + "default": default_value, } else: config_analysis[field_name] = { "type": "different", "values": values, - "default": default_value + "default": default_value, } return config_analysis @@ -1321,24 +1464,33 @@ def action_init_plate(self) -> None: # Validate all selected plates can be initialized (allow ALL failed plates to be re-initialized) invalid_plates = [] for item in selected_items: - plate_path = item['path'] + plate_path = item["path"] orchestrator = self.orchestrators.get(plate_path) # Only block plates that are currently executing - all other states can be re-initialized - if orchestrator is not None and orchestrator.state == OrchestratorState.EXECUTING: + if ( + orchestrator is not None + and orchestrator.state == OrchestratorState.EXECUTING + ): invalid_plates.append(item) if invalid_plates: - names = [item['name'] for item in invalid_plates] - logger.warning(f"Cannot initialize plates that are currently executing: {', '.join(names)}") + names = [item["name"] for item in invalid_plates] + logger.warning( + f"Cannot initialize plates that are currently executing: {', '.join(names)}" + ) return # Start async initialization self._start_async_init(selected_items, selection_mode) - def _start_async_init(self, selected_items: List[Dict], selection_mode: str) -> None: + def _start_async_init( + self, selected_items: List[Dict], selection_mode: str + ) -> None: """Start async initialization of selected plates.""" # Generate operation description - desc = self.get_operation_description(selected_items, selection_mode, "initialize") + desc = self.get_operation_description( + selected_items, selection_mode, "initialize" + ) logger.info(f"Initializing: {desc}") # Start background worker @@ -1348,12 +1500,12 @@ def _start_async_init(self, selected_items: List[Dict], selection_mode: str) -> async def _init_plates_worker(self, selected_items: List[Dict]) -> None: """Background worker for plate initialization.""" for plate_data in selected_items: - plate_path = plate_data['path'] + plate_path = plate_data["path"] # Find the actual plate in self.items (not the copy from get_selection_state) actual_plate = None for plate in self.items: - if plate['path'] == plate_path: + if plate["path"] == plate_path: actual_plate = plate break @@ -1367,10 +1519,12 @@ def init_orchestrator(): return PipelineOrchestrator( plate_path=plate_path, global_config=self.global_config, - storage_registry=self.filemanager.registry + storage_registry=self.filemanager.registry, ).initialize() - orchestrator = await asyncio.get_event_loop().run_in_executor(None, init_orchestrator) + orchestrator = await asyncio.get_event_loop().run_in_executor( + None, init_orchestrator + ) # Store orchestrator for later use (channel selection, etc.) self.orchestrators[plate_path] = orchestrator @@ -1378,32 +1532,52 @@ def init_orchestrator(): logger.info(f"Plate {actual_plate['name']} initialized successfully") except Exception as e: - logger.error(f"Failed to initialize plate {plate_path}: {e}", exc_info=True) + logger.error( + f"Failed to initialize plate {plate_path}: {e}", exc_info=True + ) # Create a failed orchestrator to track the error state failed_orchestrator = PipelineOrchestrator( plate_path=plate_path, global_config=self.global_config, - storage_registry=self.filemanager.registry + storage_registry=self.filemanager.registry, ) failed_orchestrator._state = OrchestratorState.INIT_FAILED self.orchestrators[plate_path] = failed_orchestrator - actual_plate['error'] = str(e) + actual_plate["error"] = str(e) # Trigger UI refresh after orchestrator state changes self._trigger_ui_refresh() # Update button states immediately (reactive system handles UI updates automatically) self._update_button_states() # Notify pipeline editor of status change - status_symbol = get_orchestrator_status_symbol(self.orchestrators.get(actual_plate['path'])) - self._notify_pipeline_editor_status_change(actual_plate['path'], status_symbol) + status_symbol = get_orchestrator_status_symbol( + self.orchestrators.get(actual_plate["path"]) + ) + self._notify_pipeline_editor_status_change( + actual_plate["path"], status_symbol + ) logger.debug(f"Updated plate {actual_plate['name']} status") # Final UI update (reactive system handles this automatically when self.items is modified) self._update_button_states() # Update status - success_count = len([p for p in selected_items if self.orchestrators.get(p['path']) and self.orchestrators[p['path']].state == OrchestratorState.READY]) - error_count = len([p for p in selected_items if self.orchestrators.get(p['path']) and self.orchestrators[p['path']].state == OrchestratorState.INIT_FAILED]) + success_count = len( + [ + p + for p in selected_items + if self.orchestrators.get(p["path"]) + and self.orchestrators[p["path"]].state == OrchestratorState.READY + ] + ) + error_count = len( + [ + p + for p in selected_items + if self.orchestrators.get(p["path"]) + and self.orchestrators[p["path"]].state == OrchestratorState.INIT_FAILED + ] + ) if error_count == 0: logger.info(f"Successfully initialized {success_count} plates") @@ -1422,39 +1596,59 @@ def action_compile_plate(self) -> None: # Validate all selected plates are ready for compilation (allow failed plates to be re-compiled) not_ready = [] for item in selected_items: - plate_path = item['path'] + plate_path = item["path"] orchestrator = self.orchestrators.get(plate_path) # Allow READY, COMPILE_FAILED, EXEC_FAILED, COMPILED, and COMPLETED states to be compiled/recompiled - if orchestrator is None or orchestrator.state not in [OrchestratorState.READY, OrchestratorState.COMPILE_FAILED, OrchestratorState.EXEC_FAILED, OrchestratorState.COMPILED, OrchestratorState.COMPLETED]: + if orchestrator is None or orchestrator.state not in [ + OrchestratorState.READY, + OrchestratorState.COMPILE_FAILED, + OrchestratorState.EXEC_FAILED, + OrchestratorState.COMPILED, + OrchestratorState.COMPLETED, + ]: not_ready.append(item) if not_ready: - names = [item['name'] for item in not_ready] + names = [item["name"] for item in not_ready] # More accurate error message based on actual state - if any(self.orchestrators.get(item['path']) is None for item in not_ready): - logger.warning(f"Cannot compile plates that haven't been initialized: {', '.join(names)}") - elif any(self.orchestrators.get(item['path']).state == OrchestratorState.EXECUTING for item in not_ready): - logger.warning(f"Cannot compile plates that are currently executing: {', '.join(names)}") + if any(self.orchestrators.get(item["path"]) is None for item in not_ready): + logger.warning( + f"Cannot compile plates that haven't been initialized: {', '.join(names)}" + ) + elif any( + self.orchestrators.get(item["path"]).state + == OrchestratorState.EXECUTING + for item in not_ready + ): + logger.warning( + f"Cannot compile plates that are currently executing: {', '.join(names)}" + ) else: - logger.warning(f"Cannot compile plates in current state: {', '.join(names)}") + logger.warning( + f"Cannot compile plates in current state: {', '.join(names)}" + ) return # Validate all selected plates have pipelines no_pipeline = [] for item in selected_items: - pipeline = self._get_current_pipeline_definition(item['path']) + pipeline = self._get_current_pipeline_definition(item["path"]) if not pipeline: no_pipeline.append(item) if no_pipeline: - names = [item['name'] for item in no_pipeline] - self.app.current_status = f"Cannot compile plates without pipelines: {', '.join(names)}" + names = [item["name"] for item in no_pipeline] + self.app.current_status = ( + f"Cannot compile plates without pipelines: {', '.join(names)}" + ) return # Start async compilation self._start_async_compile(selected_items, selection_mode) - def _start_async_compile(self, selected_items: List[Dict], selection_mode: str) -> None: + def _start_async_compile( + self, selected_items: List[Dict], selection_mode: str + ) -> None: """Start async compilation of selected plates.""" # Generate operation description desc = self.get_operation_description(selected_items, selection_mode, "compile") @@ -1467,12 +1661,12 @@ def _start_async_compile(self, selected_items: List[Dict], selection_mode: str) async def _compile_plates_worker(self, selected_items: List[Dict]) -> None: """Background worker for plate compilation.""" for plate_data in selected_items: - plate_path = plate_data['path'] + plate_path = plate_data["path"] # Find the actual plate in self.items (not the copy from get_selection_state) actual_plate = None for plate in self.items: - if plate['path'] == plate_path: + if plate["path"] == plate_path: actual_plate = plate break @@ -1483,7 +1677,9 @@ async def _compile_plates_worker(self, selected_items: List[Dict]) -> None: # Get definition pipeline and make fresh copy definition_pipeline = self._get_current_pipeline_definition(plate_path) if not definition_pipeline: - logger.warning(f"No pipeline defined for {actual_plate['name']}, using empty pipeline") + logger.warning( + f"No pipeline defined for {actual_plate['name']}, using empty pipeline" + ) definition_pipeline = [] try: @@ -1498,10 +1694,12 @@ def get_or_create_orchestrator(): return PipelineOrchestrator( plate_path=plate_path, global_config=self.global_config, - storage_registry=self.filemanager.registry + storage_registry=self.filemanager.registry, ).initialize() - orchestrator = await asyncio.get_event_loop().run_in_executor(None, get_or_create_orchestrator) + orchestrator = await asyncio.get_event_loop().run_in_executor( + None, get_or_create_orchestrator + ) self.orchestrators[plate_path] = orchestrator # Make fresh copy for compilation @@ -1512,12 +1710,20 @@ def get_or_create_orchestrator(): step.step_id = str(id(step)) # Ensure variable_components is never None - use FunctionStep default if step.processing_config.variable_components is None: - logger.warning(f"🔥 Step '{step.name}' has None variable_components, setting FunctionStep default") - step.processing_config.variable_components = [VariableComponents.SITE] + logger.warning( + f"🔥 Step '{step.name}' has None variable_components, setting FunctionStep default" + ) + step.processing_config.variable_components = [ + VariableComponents.SITE + ] # Also ensure it's not an empty list elif not step.processing_config.variable_components: - logger.warning(f"🔥 Step '{step.name}' has empty variable_components, setting FunctionStep default") - step.processing_config.variable_components = [VariableComponents.SITE] + logger.warning( + f"🔥 Step '{step.name}' has empty variable_components, setting FunctionStep default" + ) + step.processing_config.variable_components = [ + VariableComponents.SITE + ] # Get wells and compile (async - run in executor to avoid blocking UI) # Wrap in Pipeline object like test_main.py does @@ -1526,7 +1732,10 @@ def get_or_create_orchestrator(): # Run heavy operations in executor to avoid blocking UI # Get wells using multiprocessing axis (WELL in default config) from openhcs.constants import MULTIPROCESSING_AXIS - wells = await asyncio.get_event_loop().run_in_executor(None, lambda: orchestrator.get_component_keys(MULTIPROCESSING_AXIS)) + + wells = await asyncio.get_event_loop().run_in_executor( + None, lambda: orchestrator.get_component_keys(MULTIPROCESSING_AXIS) + ) compiled_contexts = await asyncio.get_event_loop().run_in_executor( None, orchestrator.compile_pipelines, pipeline_obj.steps, wells ) @@ -1534,21 +1743,38 @@ def get_or_create_orchestrator(): # Store state simply - no reactive property issues step_ids_in_pipeline = [id(step) for step in execution_pipeline] # Get step IDs from contexts (ProcessingContext objects) - first_well_key = list(compiled_contexts.keys())[0] if compiled_contexts else None - step_ids_in_contexts = list(compiled_contexts[first_well_key].step_plans.keys()) if first_well_key and hasattr(compiled_contexts[first_well_key], 'step_plans') else [] - logger.info(f"🔥 Storing compiled data for {plate_path}: pipeline={type(execution_pipeline)}, contexts={type(compiled_contexts)}") + first_well_key = ( + list(compiled_contexts.keys())[0] if compiled_contexts else None + ) + step_ids_in_contexts = ( + list(compiled_contexts[first_well_key].step_plans.keys()) + if first_well_key + and hasattr(compiled_contexts[first_well_key], "step_plans") + else [] + ) + logger.info( + f"🔥 Storing compiled data for {plate_path}: pipeline={type(execution_pipeline)}, contexts={type(compiled_contexts)}" + ) logger.info(f"🔥 Step IDs in pipeline: {step_ids_in_pipeline}") logger.info(f"🔥 Step IDs in contexts: {step_ids_in_contexts}") - self.plate_compiled_data[plate_path] = (execution_pipeline, compiled_contexts) - logger.info(f"🔥 Stored! Available compiled plates: {list(self.plate_compiled_data.keys())}") + self.plate_compiled_data[plate_path] = ( + execution_pipeline, + compiled_contexts, + ) + logger.info( + f"🔥 Stored! Available compiled plates: {list(self.plate_compiled_data.keys())}" + ) # Orchestrator state is already set to COMPILED by compile_pipelines() method logger.info(f"🔥 Successfully compiled {plate_path}") except Exception as e: - logger.error(f"🔥 COMPILATION ERROR: Pipeline compilation failed for {plate_path}: {e}", exc_info=True) + logger.error( + f"🔥 COMPILATION ERROR: Pipeline compilation failed for {plate_path}: {e}", + exc_info=True, + ) # Orchestrator state is already set to FAILED by compile_pipelines() method - actual_plate['error'] = str(e) + actual_plate["error"] = str(e) # Don't store anything in plate_compiled_data on failure # Trigger UI refresh after orchestrator state changes @@ -1556,15 +1782,34 @@ def get_or_create_orchestrator(): # Update button states immediately (reactive system handles UI updates automatically) self._update_button_states() # Notify pipeline editor of status change - status_symbol = get_orchestrator_status_symbol(self.orchestrators.get(actual_plate['path'])) - self._notify_pipeline_editor_status_change(actual_plate['path'], status_symbol) + status_symbol = get_orchestrator_status_symbol( + self.orchestrators.get(actual_plate["path"]) + ) + self._notify_pipeline_editor_status_change( + actual_plate["path"], status_symbol + ) # Final UI update (reactive system handles this automatically when self.items is modified) self._update_button_states() # Update status - success_count = len([p for p in selected_items if self.orchestrators.get(p['path']) and self.orchestrators[p['path']].state == OrchestratorState.COMPILED]) - error_count = len([p for p in selected_items if self.orchestrators.get(p['path']) and self.orchestrators[p['path']].state == OrchestratorState.COMPILE_FAILED]) + success_count = len( + [ + p + for p in selected_items + if self.orchestrators.get(p["path"]) + and self.orchestrators[p["path"]].state == OrchestratorState.COMPILED + ] + ) + error_count = len( + [ + p + for p in selected_items + if self.orchestrators.get(p["path"]) + and self.orchestrators[p["path"]].state + == OrchestratorState.COMPILE_FAILED + ] + ) if error_count == 0: logger.info(f"Successfully compiled {success_count} plates") @@ -1582,15 +1827,17 @@ async def action_code_plate(self) -> None: try: # Get pipeline data for selected plates - plate_paths = [item['path'] for item in selected_items] + plate_paths = [item["path"] for item in selected_items] pipeline_data = {} # Collect pipeline steps for each plate for plate_path in plate_paths: - if hasattr(self, 'pipeline_editor') and self.pipeline_editor: + if hasattr(self, "pipeline_editor") and self.pipeline_editor: # Get pipeline steps from pipeline editor if available if plate_path in self.pipeline_editor.plate_pipelines: - pipeline_data[plate_path] = self.pipeline_editor.plate_pipelines[plate_path] + pipeline_data[plate_path] = ( + self.pipeline_editor.plate_pipelines[plate_path] + ) else: pipeline_data[plate_path] = [] else: @@ -1601,27 +1848,34 @@ async def action_code_plate(self) -> None: # Create data structure the serializer expects data = { - 'plate_paths': plate_paths, - 'pipeline_data': pipeline_data, - 'global_config': self.app.global_config + "plate_paths": plate_paths, + "pipeline_data": pipeline_data, + "global_config": self.app.global_config, } # Extract variables from data dict - plate_paths = data['plate_paths'] - pipeline_data = data['pipeline_data'] + plate_paths = data["plate_paths"] + pipeline_data = data["pipeline_data"] # Generate just the orchestrator configuration (no execution wrapper) import openhcs.serialization.pycodify_formatters # noqa: F401 - from pycodify import Assignment, BlankLine, CodeBlock, generate_python_source + from pycodify import ( + Assignment, + BlankLine, + CodeBlock, + generate_python_source, + ) python_code = generate_python_source( - CodeBlock.from_items([ - Assignment("plate_paths", plate_paths), - BlankLine(), - Assignment("global_config", self.app.global_config), - BlankLine(), - Assignment("pipeline_data", pipeline_data), - ]), + CodeBlock.from_items( + [ + Assignment("plate_paths", plate_paths), + BlankLine(), + Assignment("global_config", self.app.global_config), + BlankLine(), + Assignment("pipeline_data", pipeline_data), + ] + ), header="# Edit this orchestrator configuration and save to apply changes", clean_mode=True, # Default to clean mode - only show non-default values ) @@ -1635,54 +1889,75 @@ def handle_edited_code(edited_code: str): exec(edited_code, namespace) # Update pipeline data if present (composition: orchestrator contains pipelines) - if 'pipeline_data' in namespace: - new_pipeline_data = namespace['pipeline_data'] + if "pipeline_data" in namespace: + new_pipeline_data = namespace["pipeline_data"] # Update pipeline editor using reactive system (like pipeline code button does) - if hasattr(self, 'pipeline_editor') and self.pipeline_editor: + if hasattr(self, "pipeline_editor") and self.pipeline_editor: # Update plate pipelines storage - current_pipelines = dict(self.pipeline_editor.plate_pipelines) + current_pipelines = dict( + self.pipeline_editor.plate_pipelines + ) current_pipelines.update(new_pipeline_data) self.pipeline_editor.plate_pipelines = current_pipelines # If current plate is in the edited data, update the current view too current_plate = self.pipeline_editor.current_plate if current_plate and current_plate in new_pipeline_data: - self.pipeline_editor.pipeline_steps = new_pipeline_data[current_plate] + self.pipeline_editor.pipeline_steps = new_pipeline_data[ + current_plate + ] - self.app.current_status = f"Pipeline data updated for {len(new_pipeline_data)} plates" + self.app.current_status = ( + f"Pipeline data updated for {len(new_pipeline_data)} plates" + ) # Update global config if present - elif 'global_config' in namespace: - new_global_config = namespace['global_config'] + elif "global_config" in namespace: + new_global_config = namespace["global_config"] import asyncio + for plate_path in plate_paths: if plate_path in self.orchestrators: orchestrator = self.orchestrators[plate_path] - asyncio.create_task(orchestrator.apply_new_global_config(new_global_config)) - self.app.current_status = f"Global config updated for {len(plate_paths)} plates" + asyncio.create_task( + orchestrator.apply_new_global_config( + new_global_config + ) + ) + self.app.current_status = ( + f"Global config updated for {len(plate_paths)} plates" + ) # Update orchestrators list if present - elif 'orchestrators' in namespace: - new_orchestrators = namespace['orchestrators'] + elif "orchestrators" in namespace: + new_orchestrators = namespace["orchestrators"] self.app.current_status = f"Orchestrator list updated with {len(new_orchestrators)} orchestrators" else: - self.app.show_error("Parse Error", "No valid assignments found in edited code") + self.app.show_error( + "Parse Error", "No valid assignments found in edited code" + ) except SyntaxError as e: self.app.show_error("Syntax Error", f"Invalid Python syntax: {e}") except Exception as e: import traceback + full_traceback = traceback.format_exc() - logger.error(f"Failed to parse edited orchestrator code: {e}\nFull traceback:\n{full_traceback}") - self.app.show_error("Edit Error", f"Failed to parse orchestrator code: {str(e)}\n\nFull traceback:\n{full_traceback}") + logger.error( + f"Failed to parse edited orchestrator code: {e}\nFull traceback:\n{full_traceback}" + ) + self.app.show_error( + "Edit Error", + f"Failed to parse orchestrator code: {str(e)}\n\nFull traceback:\n{full_traceback}", + ) # Launch terminal editor launcher = TerminalLauncher(self.app) await launcher.launch_editor_for_file( file_content=python_code, - file_extension='.py', - on_save_callback=handle_edited_code + file_extension=".py", + on_save_callback=handle_edited_code, ) except Exception as e: @@ -1700,15 +1975,17 @@ async def action_save_python_script(self) -> None: try: # Get pipeline data for selected plates - plate_paths = [item['path'] for item in selected_items] + plate_paths = [item["path"] for item in selected_items] pipeline_data = {} # Collect pipeline steps for each plate for plate_path in plate_paths: - if hasattr(self, 'pipeline_editor') and self.pipeline_editor: + if hasattr(self, "pipeline_editor") and self.pipeline_editor: # Get pipeline steps from pipeline editor if available if plate_path in self.pipeline_editor.plate_pipelines: - pipeline_data[plate_path] = self.pipeline_editor.plate_pipelines[plate_path] + pipeline_data[plate_path] = ( + self.pipeline_editor.plate_pipelines[plate_path] + ) else: pipeline_data[plate_path] = [] else: @@ -1716,16 +1993,19 @@ async def action_save_python_script(self) -> None: # Create data structure the serializer expects data = { - 'plate_paths': plate_paths, - 'pipeline_data': pipeline_data, - 'global_config': self.app.global_config + "plate_paths": plate_paths, + "pipeline_data": pipeline_data, + "global_config": self.app.global_config, } # Generate complete executable Python script using the serializer python_code = self._generate_executable_script(data) # Launch file browser to save the script - from openhcs.textual_tui.windows.file_browser_window import open_file_browser_window, BrowserMode + from openhcs.textual_tui.windows.file_browser_window import ( + open_file_browser_window, + BrowserMode, + ) from openhcs.textual_tui.services.file_browser_service import SelectionMode from openhcs.core.path_cache import get_cached_browser_path, PathCacheKey from openhcs.constants.constants import Backend @@ -1742,11 +2022,13 @@ def handle_save_result(result): if save_path: try: # Write the Python script to the selected file - with open(save_path, 'w') as f: + with open(save_path, "w") as f: f.write(python_code) logger.info(f"Python script saved to: {save_path}") - self.app.current_status = f"Python script saved to: {save_path}" + self.app.current_status = ( + f"Python script saved to: {save_path}" + ) except Exception as e: logger.error(f"Failed to save Python script: {e}") self.app.current_status = f"Failed to save script: {e}" @@ -1763,11 +2045,11 @@ def handle_save_result(result): title="Save Python Pipeline Script", mode=BrowserMode.SAVE, selection_mode=SelectionMode.FILES_ONLY, - filter_extensions=['.py'], + filter_extensions=[".py"], default_filename=default_filename, cache_key=PathCacheKey.PIPELINE_FILES, on_result_callback=handle_save_result, - caller_id="plate_manager_save_script" + caller_id="plate_manager_save_script", ) except Exception as e: @@ -1781,12 +2063,16 @@ def _generate_executable_script(self, data: Dict) -> str: from openhcs.debug.pickle_converter import convert_pickle_to_python # Create temporary pickle file - with tempfile.NamedTemporaryFile(mode='wb', suffix='.pkl', delete=False) as temp_pickle: + with tempfile.NamedTemporaryFile( + mode="wb", suffix=".pkl", delete=False + ) as temp_pickle: pickle.dump(data, temp_pickle) temp_pickle_path = temp_pickle.name # Create temporary output file - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as temp_output: + with tempfile.NamedTemporaryFile( + mode="w", suffix=".py", delete=False + ) as temp_output: temp_output_path = temp_output.name try: @@ -1794,7 +2080,7 @@ def _generate_executable_script(self, data: Dict) -> str: convert_pickle_to_python(temp_pickle_path, temp_output_path) # Read the generated script - with open(temp_output_path, 'r') as f: + with open(temp_output_path, "r") as f: script_content = f.read() return script_content @@ -1802,6 +2088,7 @@ def _generate_executable_script(self, data: Dict) -> str: finally: # Clean up temp files import os + try: os.unlink(temp_pickle_path) os.unlink(temp_output_path) diff --git a/openhcs/textual_tui/windows/function_selector_window.py b/openhcs/textual_tui/windows/function_selector_window.py index bd719d4bd..0b0c40e9e 100644 --- a/openhcs/textual_tui/windows/function_selector_window.py +++ b/openhcs/textual_tui/windows/function_selector_window.py @@ -236,8 +236,12 @@ def _populate_table(self, table: DataTable, functions_metadata: Dict[str, Functi description = metadata.doc[:50] + "..." if len(metadata.doc) > 50 else metadata.doc # Add row with function metadata - Backend shows memory type, Registry shows source + # Display original_name (just the function name) instead of name (which includes module prefix) + # The module is shown separately in the Module column + display_name = metadata.original_name or metadata.name + row_key = table.add_row( - metadata.name, + display_name, metadata.module.split('.')[-1] if metadata.module else "unknown", # Show only last part of module memory_type.title(), # Show actual memory type (cupy, numpy, etc.) registry_name.title(), # Show registry source (openhcs, skimage, etc.) diff --git a/openhcs/ui/shared/streaming_service.py b/openhcs/ui/shared/streaming_service.py index f9af46a4e..0151f06f9 100644 --- a/openhcs/ui/shared/streaming_service.py +++ b/openhcs/ui/shared/streaming_service.py @@ -3,14 +3,16 @@ Eliminates duplication between Napari and Fiji streaming code by parametrizing on viewer_type. All heavy operations run in background threads. """ + from __future__ import annotations import logging -import threading import time from pathlib import Path from typing import TYPE_CHECKING, Callable, Literal +from objectstate import spawn_thread_with_context + if TYPE_CHECKING: from polystore.filemanager import FileManager @@ -20,16 +22,18 @@ # Each image creates a shared memory segment (file descriptor on Linux) CHUNK_SIZE = 50 -ViewerType = Literal['napari', 'fiji'] +# ViewerType is now the registry key (e.g., 'napari_streaming_config', 'fiji_streaming_config') +# This provides type safety and consistency throughout the codebase +ViewerType = str # Registry key from StreamingConfig.__registry__ class StreamingService: """Unified service for streaming images/ROIs to viewers. - + Handles all viewer communication in background threads. Uses callbacks for UI thread communication (status updates, errors). """ - + def __init__( self, filemanager: FileManager, @@ -39,12 +43,49 @@ def __init__( self.filemanager = filemanager self.microscope_handler = microscope_handler self.plate_path = plate_path - - def _get_backend_enum(self, viewer_type: ViewerType): - """Get the backend enum for the viewer type.""" + + @staticmethod + def _get_display_name(viewer_type: str) -> str: + """Get display name from registry key. + + Args: + viewer_type: Registry key (e.g., 'napari_streaming_config') + + Returns: + Display name (e.g., 'Napari') + """ + # Extract viewer name from registry key (e.g., 'napari_streaming_config' -> 'napari') + viewer_name = viewer_type.replace('_streaming_config', '') + return viewer_name.title() + + @classmethod + def supported_viewer_types(cls): + """Return a list of supported viewer short names (e.g. 'napari', 'fiji'). + + Centralized so UI can discover which viewer buttons to create instead + of hardcoding Napari/Fiji in multiple places. + """ + # Use the metaclass-driven StreamingConfig registry. + # StreamingConfig uses AutoRegisterMeta with __registry_key__ = 'viewer_type' + from openhcs.core.config import StreamingConfig + + # Return stable ordering from the registry keys + return sorted(list(StreamingConfig.__registry__.keys())) + + def _get_backend_enum(self, viewer_type: str): + """Get the backend enum for the viewer type. + + Args: + viewer_type: Registry key (e.g., 'napari_streaming_config', 'fiji_streaming_config') + """ from openhcs.constants.constants import Backend as BackendEnum - return BackendEnum.NAPARI_STREAM if viewer_type == 'napari' else BackendEnum.FIJI_STREAM - + + return ( + BackendEnum.NAPARI_STREAM + if "napari" in viewer_type + else BackendEnum.FIJI_STREAM + ) + def _wait_for_viewer_ready( self, viewer, @@ -52,35 +93,46 @@ def _wait_for_viewer_ready( num_items: int, ) -> None: """Wait for viewer to be ready, registering as launching if needed.""" - from openhcs.pyqt_gui.widgets.shared.zmq_server_manager import ( - register_launching_viewer, unregister_launching_viewer - ) - + # Use centralized ViewerStateManager for launching/queued state + from zmqruntime.viewer_state import ViewerStateManager + + manager = ViewerStateManager.get_instance() + is_already_running = viewer.wait_for_ready(timeout=0.1) - + + # Update queued images for UI display via manager. The QueueTracker + # will later update counts precisely as images are sent/acked. + manager.update_queued_images(viewer_type, viewer.port, num_items) + if not is_already_running: - register_launching_viewer(viewer.port, viewer_type, num_items) - logger.info(f"Registered launching {viewer_type} viewer on port {viewer.port}") - + display_name = self._get_display_name(viewer_type) + logger.info( + f"Waiting for {display_name} viewer on port {viewer.port} to become ready" + ) + if not viewer.wait_for_ready(timeout=15.0): - unregister_launching_viewer(viewer.port) - raise RuntimeError(f"{viewer_type.capitalize()} viewer on port {viewer.port} failed to become ready") - - logger.info(f"{viewer_type.capitalize()} viewer on port {viewer.port} is ready") - unregister_launching_viewer(viewer.port) - else: - logger.info(f"{viewer_type.capitalize()} viewer on port {viewer.port} is already running") - + # Clear queued count for UI if startup failed + manager.update_queued_images(viewer_type, viewer.port, 0) + raise RuntimeError( + f"{display_name} viewer on port {viewer.port} failed to become ready" + ) + + logger.info( + f"{display_name} viewer on port {viewer.port} is ready" + ) + def _build_metadata(self, viewer, config, source: str) -> dict: """Build metadata dict for streaming.""" return { - 'port': viewer.port, - 'display_config': config, - 'microscope_handler': self.microscope_handler, - 'plate_path': self.plate_path, - 'source': source, + "port": viewer.port, + "host": config.host, + "transport_mode": config.transport_mode, + "display_config": config, + "microscope_handler": self.microscope_handler, + "plate_path": self.plate_path, + "source": source, } - + def stream_images_async( self, viewer, @@ -93,59 +145,75 @@ def stream_images_async( error_callback: Callable[[str], None], ) -> None: """Load and stream images to viewer in background thread. - + Uses chunked streaming to prevent file descriptor exhaustion. """ backend_enum = self._get_backend_enum(viewer_type) - + display_name = self._get_display_name(viewer_type) + def _worker(): try: self._wait_for_viewer_ready(viewer, viewer_type, len(filenames)) - + total_images = len(filenames) num_chunks = (total_images + CHUNK_SIZE - 1) // CHUNK_SIZE logger.info(f"Streaming {total_images} images in {num_chunks} chunks") - + for chunk_idx in range(num_chunks): start_idx = chunk_idx * CHUNK_SIZE end_idx = min(start_idx + CHUNK_SIZE, total_images) chunk_filenames = filenames[start_idx:end_idx] - - status_callback(f"Loading chunk {chunk_idx + 1}/{num_chunks} ({len(chunk_filenames)} images)...") - + + status_callback( + f"Loading chunk {chunk_idx + 1}/{num_chunks} ({len(chunk_filenames)} images)..." + ) + # Load chunk image_data_list = [] file_paths = [] for filename in chunk_filenames: image_path = plate_path / filename - image_data = self.filemanager.load(str(image_path), read_backend) + image_data = self.filemanager.load( + str(image_path), read_backend + ) image_data_list.append(image_data) file_paths.append(filename) - - logger.info(f"Loaded chunk {chunk_idx + 1}/{num_chunks}: {len(image_data_list)} images") - - source = Path(file_paths[0]).parent.name if file_paths else 'unknown_source' + + logger.info( + f"Loaded chunk {chunk_idx + 1}/{num_chunks}: {len(image_data_list)} images" + ) + + source = ( + Path(file_paths[0]).parent.name + if file_paths + else "unknown_source" + ) metadata = self._build_metadata(viewer, config, source) - + self.filemanager.save_batch( image_data_list, file_paths, backend_enum.value, **metadata ) - logger.info(f"Streamed chunk {chunk_idx + 1}/{num_chunks} to {viewer_type}") - + logger.info( + f"Streamed chunk {chunk_idx + 1}/{num_chunks} to {display_name}" + ) + if chunk_idx < num_chunks - 1: time.sleep(0.1) - - logger.info(f"Successfully streamed {total_images} images to {viewer_type}") - status_callback(f"Streamed {total_images} images to {viewer_type.capitalize()}") - + + logger.info( + f"Successfully streamed {total_images} images to {display_name}" + ) + status_callback( + f"Streamed {total_images} images to {display_name}" + ) + except Exception as e: - logger.error(f"Failed to stream images to {viewer_type}: {e}") + logger.error(f"Failed to stream images to {display_name}: {e}") status_callback(f"Error: {e}") error_callback(str(e)) - - thread = threading.Thread(target=_worker, daemon=True) - thread.start() - logger.info(f"Started streaming {len(filenames)} images to {viewer_type}") + + spawn_thread_with_context(_worker, name=f"stream_images_{viewer_type}") + logger.info(f"Started streaming {len(filenames)} images to {display_name}") def stream_rois_async( self, @@ -159,6 +227,7 @@ def stream_rois_async( ) -> None: """Load and stream ROI files to viewer in background thread.""" backend_enum = self._get_backend_enum(viewer_type) + display_name = self._get_display_name(viewer_type) def _worker(): try: @@ -197,18 +266,21 @@ def _worker(): source = Path(paths[0]).parent.name if paths else "unknown_source" metadata = self._build_metadata(viewer, config, source) - status_callback(f"Streaming {len(paths)} ROI file(s) to {viewer_type.capitalize()}...") + status_callback( + f"Streaming {len(paths)} ROI file(s) to {display_name}..." + ) - self.filemanager.save_batch(data_list, paths, backend_enum.value, **metadata) + self.filemanager.save_batch( + data_list, paths, backend_enum.value, **metadata + ) - msg = f"Streamed {len(paths)} ROI file(s) to {viewer_type.capitalize()} on port {viewer.port}" + msg = f"Streamed {len(paths)} ROI file(s) to {display_name} on port {viewer.port}" logger.info(msg) status_callback(msg) except Exception as e: - logger.error(f"Failed to stream ROIs to {viewer_type}: {e}") + logger.error(f"Failed to stream ROIs to {display_name}: {e}") status_callback(f"Error: {e}") error_callback(str(e)) - thread = threading.Thread(target=_worker, daemon=True) - thread.start() + spawn_thread_with_context(_worker, name=f"stream_rois_{viewer_type}") diff --git a/openhcs/utils/performance_monitor.py b/openhcs/utils/performance_monitor.py index 2b581c9e2..eee624eb0 100644 --- a/openhcs/utils/performance_monitor.py +++ b/openhcs/utils/performance_monitor.py @@ -12,39 +12,41 @@ from pathlib import Path # Create performance logger -perf_logger = logging.getLogger('openhcs.performance') -perf_logger.setLevel(logging.DEBUG) +perf_logger = logging.getLogger("openhcs.performance") +perf_logger.setLevel(logging.WARNING) # Add file handler for performance logs -perf_log_file = Path.home() / '.local' / 'share' / 'openhcs' / 'logs' / 'performance.log' +perf_log_file = ( + Path.home() / ".local" / "share" / "openhcs" / "logs" / "performance.log" +) perf_log_file.parent.mkdir(parents=True, exist_ok=True) file_handler = logging.FileHandler(perf_log_file) file_handler.setLevel(logging.DEBUG) -file_handler.setFormatter(logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s' -)) +file_handler.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +) perf_logger.addHandler(file_handler) # Also log to console console_handler = logging.StreamHandler() -console_handler.setLevel(logging.DEBUG) -console_handler.setFormatter(logging.Formatter( - '⏱️ %(message)s' -)) +console_handler.setLevel(logging.WARNING) +console_handler.setFormatter(logging.Formatter("⏱️ %(message)s")) perf_logger.addHandler(console_handler) @contextmanager -def timer(operation_name: str, threshold_ms: float = 0.0, log_args: bool = False, **kwargs): +def timer( + operation_name: str, threshold_ms: float = 0.0, log_args: bool = False, **kwargs +): """Context manager for timing operations. - + Args: operation_name: Name of the operation being timed threshold_ms: Only log if operation takes longer than this (in milliseconds) log_args: Whether to log kwargs in the message **kwargs: Additional context to include in log message - + Example: with timer("Loading config", threshold_ms=10.0, config_type="GlobalPipelineConfig"): config = load_config() @@ -54,33 +56,34 @@ def timer(operation_name: str, threshold_ms: float = 0.0, log_args: bool = False yield finally: elapsed_ms = (time.perf_counter() - start) * 1000 - + if elapsed_ms >= threshold_ms: msg = f"{operation_name}: {elapsed_ms:.2f}ms" if log_args and kwargs: args_str = ", ".join(f"{k}={v}" for k, v in kwargs.items()) msg += f" ({args_str})" - + perf_logger.debug(msg) def timed(operation_name: Optional[str] = None, threshold_ms: float = 0.0): """Decorator for timing function calls. - + Args: operation_name: Name for the operation (defaults to function name) threshold_ms: Only log if operation takes longer than this (in milliseconds) - + Example: @timed("Config loading", threshold_ms=10.0) def load_config(): ... """ + def decorator(func: Callable) -> Callable: nonlocal operation_name if operation_name is None: operation_name = f"{func.__module__}.{func.__qualname__}" - + @functools.wraps(func) def wrapper(*args, **kwargs): start = time.perf_counter() @@ -88,32 +91,33 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) finally: elapsed_ms = (time.perf_counter() - start) * 1000 - + if elapsed_ms >= threshold_ms: perf_logger.debug(f"{operation_name}: {elapsed_ms:.2f}ms") - + return wrapper + return decorator class PerformanceMonitor: """Accumulates timing statistics for repeated operations. - + Example: monitor = PerformanceMonitor("Placeholder resolution") - + for field in fields: with monitor.measure(): resolve_placeholder(field) - + monitor.report() # Logs summary statistics """ - + def __init__(self, operation_name: str): self.operation_name = operation_name self.timings = [] self.current_start = None - + @contextmanager def measure(self): """Measure a single operation.""" @@ -123,23 +127,23 @@ def measure(self): finally: elapsed_ms = (time.perf_counter() - start) * 1000 self.timings.append(elapsed_ms) - + def report(self, log_individual: bool = False): """Log summary statistics. - + Args: log_individual: Whether to log each individual timing """ if not self.timings: perf_logger.debug(f"{self.operation_name}: No measurements") return - + count = len(self.timings) total_ms = sum(self.timings) avg_ms = total_ms / count min_ms = min(self.timings) max_ms = max(self.timings) - + perf_logger.debug( f"{self.operation_name} - " f"Count: {count}, " @@ -148,11 +152,11 @@ def report(self, log_individual: bool = False): f"Min: {min_ms:.2f}ms, " f"Max: {max_ms:.2f}ms" ) - + if log_individual: for i, timing in enumerate(self.timings, 1): perf_logger.debug(f" #{i}: {timing:.2f}ms") - + def reset(self): """Clear all timings.""" self.timings.clear() @@ -164,7 +168,7 @@ def reset(self): def get_monitor(operation_name: str) -> PerformanceMonitor: """Get or create a global monitor for an operation. - + Example: monitor = get_monitor("Placeholder resolution") with monitor.measure(): @@ -180,14 +184,14 @@ def report_all_monitors(): if not _monitors: perf_logger.debug("No performance monitors active") return - + perf_logger.debug("=" * 60) perf_logger.debug("PERFORMANCE SUMMARY") perf_logger.debug("=" * 60) - + for monitor in _monitors.values(): monitor.report() - + perf_logger.debug("=" * 60) diff --git a/pyproject.toml b/pyproject.toml index 6eb1a85f6..729ad763f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ dependencies = [ # Core utilities "setuptools", + "packaging>=21.0", # For version parsing (replaces deprecated pkg_resources) "watchdog>=6.0.0", "portalocker>=2.8.2", # Cross-platform file locking (Windows compatibility for fcntl) "requests>=2.31.0", # HTTP library for LLM service communication @@ -78,7 +79,7 @@ dependencies = [ "zmqruntime>=0.1.0", "pycodify>=0.1.0", "objectstate>=0.1.0", - "python-introspect>=0.1.0", + "python-introspect>=0.1.1", "metaclass-registry>=0.1.0", "arraybridge>=0.1.0", "polystore>=0.1.0", @@ -140,12 +141,14 @@ viz = [ # OMERO integration dependencies # -# IMPORTANT: zeroc-ice is a dependency of omero-py but is NOT available on PyPI. -# We use direct URL references with environment markers to install the appropriate -# zeroc-ice wheel from Glencoe Software's repository. +# IMPORTANT: PyPI does not allow direct URL dependencies in project metadata. +# The zeroc-ice wheels required by omero-py must be installed via the helper script +# or requirements file instead of as an extra dependency here. # -# Installation (automatic): -# pip install 'openhcs[omero]' +# Installation: +# python scripts/install_omero_deps.py +# # or +# pip install -r requirements-omero.txt # # Alternative installation (using requirements file): # pip install -r requirements-omero.txt @@ -155,22 +158,6 @@ viz = [ # - scripts/install_omero_deps.py # - requirements-omero.txt omero = [ - # Linux x86_64 - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20240202/zeroc_ice-3.6.5-cp311-cp311-manylinux_2_28_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.11'", - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20240202/zeroc_ice-3.6.5-cp312-cp312-manylinux_2_28_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.12'", - - # Linux aarch64 - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-aarch64/releases/download/20240621/zeroc_ice-3.6.5-cp311-cp311-manylinux_2_28_aarch64.whl ; sys_platform == 'linux' and platform_machine == 'aarch64' and python_version == '3.11'", - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-aarch64/releases/download/20240621/zeroc_ice-3.6.5-cp312-cp312-manylinux_2_28_aarch64.whl ; sys_platform == 'linux' and platform_machine == 'aarch64' and python_version == '3.12'", - - # macOS universal2 - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-macos/releases/download/20231208/zeroc_ice-3.6.5-cp311-cp311-macosx_11_0_universal2.whl ; sys_platform == 'darwin' and python_version == '3.11'", - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-macos/releases/download/20231208/zeroc_ice-3.6.5-cp312-cp312-macosx_11_0_universal2.whl ; sys_platform == 'darwin' and python_version == '3.12'", - - # Windows amd64 - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-windows-x86_64/releases/download/20240326/zeroc_ice-3.6.5-cp311-cp311-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.11'", - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-windows-x86_64/releases/download/20240326/zeroc_ice-3.6.5-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12'", - "omero-py>=5.19.0", ] @@ -184,31 +171,13 @@ docs = [ # Remote execution and OMERO integration # -# IMPORTANT: zeroc-ice is a dependency of omero-py but is NOT available on PyPI. -# We use direct URL references with environment markers to install the appropriate -# zeroc-ice wheel from Glencoe Software's repository. +# IMPORTANT: PyPI does not allow direct URL dependencies in project metadata. +# Install zeroc-ice for OMERO using scripts/install_omero_deps.py or requirements-omero.txt. # # See the 'omero' optional dependency section above for more details. remote = [ "pyzmq>=26.0.0", # ZMQ-based remote execution and streaming "omero-py>=5.19.0", # OMERO Python bindings for OMERO backend - - # Duplicate zeroc-ice URLs (extras can't reference other extras) - # Linux x86_64 - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20240202/zeroc_ice-3.6.5-cp311-cp311-manylinux_2_28_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.11'", - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20240202/zeroc_ice-3.6.5-cp312-cp312-manylinux_2_28_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.12'", - - # Linux aarch64 - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-aarch64/releases/download/20240621/zeroc_ice-3.6.5-cp311-cp311-manylinux_2_28_aarch64.whl ; sys_platform == 'linux' and platform_machine == 'aarch64' and python_version == '3.11'", - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-aarch64/releases/download/20240621/zeroc_ice-3.6.5-cp312-cp312-manylinux_2_28_aarch64.whl ; sys_platform == 'linux' and platform_machine == 'aarch64' and python_version == '3.12'", - - # macOS universal2 - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-macos/releases/download/20231208/zeroc_ice-3.6.5-cp311-cp311-macosx_11_0_universal2.whl ; sys_platform == 'darwin' and python_version == '3.11'", - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-macos/releases/download/20231208/zeroc_ice-3.6.5-cp312-cp312-macosx_11_0_universal2.whl ; sys_platform == 'darwin' and python_version == '3.12'", - - # Windows amd64 - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-windows-x86_64/releases/download/20240326/zeroc_ice-3.6.5-cp311-cp311-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.11'", - "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-windows-x86_64/releases/download/20240326/zeroc_ice-3.6.5-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12'", ] # GPU acceleration dependencies @@ -276,10 +245,10 @@ all = [ ] [project.urls] -Homepage = "https://github.com/trissim/openhcs" -"Bug Reports" = "https://github.com/trissim/openhcs/issues" -Source = "https://github.com/trissim/openhcs" -Documentation = "https://github.com/trissim/openhcs/blob/main/README.md" +Homepage = "https://github.com/OpenHCSDev/OpenHCS" +"Bug Reports" = "https://github.com/OpenHCSDev/OpenHCS/issues" +Source = "https://github.com/OpenHCSDev/OpenHCS" +Documentation = "https://github.com/OpenHCSDev/OpenHCS/blob/main/README.md" [project.scripts] # Main interfaces - GUI is now the default diff --git a/scripts/figures/README.md b/scripts/figures/README.md new file mode 100644 index 000000000..1c6ab9579 --- /dev/null +++ b/scripts/figures/README.md @@ -0,0 +1,127 @@ +# Figure Generation Scripts + +This directory contains scripts for generating various figures, plots, and visualizations from imaging data and analysis results. + +## Scripts + +### `generate_xy_xz_composite_figures.py` +Creates composite figures showing XY, XZ, and YZ max projections for each well. + +**Features:** +- Pairs XY, XZ, and YZ max projection images +- Creates figures with a two-column layout (XY left, XZ/YZ stacked right) +- Includes well identification in titles +- Optional metadata mapping to append labels to titles +- Automatic column labels (HA-OXIME/matrigel with Time A/B/C) +- Optional well notation toggle for titles/filenames (R##C## or A01) +- Optional tiled mosaic output aligned by well position +- Output filenames include group labels when available + +**Usage:** +```bash +python scripts/figures/generate_xy_xz_composite_figures.py \ + --input-dir /path/to/max_project \ + --output-dir /path/to/output + +# Optional metadata mapping +python scripts/figures/generate_xy_xz_composite_figures.py \ + --input-dir /path/to/max_project \ + --output-dir /path/to/output \ + --metadata-csv /path/to/metadata.csv + +# Optional tiled mosaic output +python scripts/figures/generate_xy_xz_composite_figures.py \ + --input-dir /path/to/max_project \ + --output-dir /path/to/output \ + --mosaic-output /path/to/output/mosaic.png + +# Optional well notation toggle +python scripts/figures/generate_xy_xz_composite_figures.py \ + --input-dir /path/to/max_project \ + --output-dir /path/to/output \ + --well-format a01 +``` + +--- + +### `generate_figure_overlays.py` +Generates figure overlays showing analyzed wells with ROIs (Regions of Interest) overlaid on images. + +**Features:** +- Matches ROI files (.roi.zip) to corresponding images +- Filters wells based on analysis configuration (excludes excluded wells) +- Overlays polygon ROIs with labels on images +- Supports multiprocessing for faster processing +- Includes channel information and well annotations + +**Usage:** +```bash +python scripts/figures/generate_figure_overlays.py \ + --plate-dir /path/to/plate1 \ + --plate-dir /path/to/plate2 \ + --config /path/to/config.xlsx \ + --results /path/to/results.csv +``` + +--- + +### `generate_automated_plots.py` +Generates automated bar plots with statistical analysis from compiled results files. + +**Features:** +- Reads plot configuration from config.xlsx (plot_config sheet) +- Creates bar plots with error bars (SEM) +- Includes scatter points for individual data points +- Runs statistical tests (ANOVA, pairwise t-tests) +- Adds significance markers (*, **, ***, ns) +- Processes three data types: normalized, raw per replicate, and raw per well + +**Usage:** +```bash +python scripts/figures/generate_automated_plots.py \ + --config /path/to/config.xlsx \ + --results-dir /path/to/compiled_results \ + --output-dir /path/to/plots +``` + +--- + +### `generate_figure_mosaics.py` +Creates mosaic/grid images from individual figure overlays, grouping by condition. + +**Features:** +- Groups figures by experimental condition +- Creates grid layouts (roughly square) +- Sorts images by channel and replicate +- Outputs high-resolution mosaics (300 DPI) + +**Usage:** +```bash +python scripts/figures/generate_figure_mosaics.py \ + --figures-dir /path/to/figures \ + --output-dir /path/to/mosaics +``` + +## Workflow + +Typical figure generation workflow: + +1. **Generate overlays**: Use `generate_figure_overlays.py` to create individual well figures with ROIs +2. **Create composites**: Use `generate_xy_xz_composite_figures.py` for projection comparisons +3. **Build mosaics**: Use `generate_figure_mosaics.py` to create condition summary grids +4. **Plot statistics**: Use `generate_automated_plots.py` for quantitative analysis plots + +## Dependencies + +All scripts require: +- Python 3.x +- matplotlib +- numpy +- PIL/Pillow +- pandas (for automated plots) +- scipy (for statistical tests) +- openhcs package (for ROI handling) + +## Related + +See also `../napari-plugins/` for napari plugins for creating orthogonal projections and exporting layers directly from napari. diff --git a/scripts/generate_automated_plots.py b/scripts/figures/generate_automated_plots.py similarity index 100% rename from scripts/generate_automated_plots.py rename to scripts/figures/generate_automated_plots.py diff --git a/scripts/generate_figure_mosaics.py b/scripts/figures/generate_figure_mosaics.py similarity index 100% rename from scripts/generate_figure_mosaics.py rename to scripts/figures/generate_figure_mosaics.py diff --git a/scripts/generate_figure_overlays.py b/scripts/figures/generate_figure_overlays.py similarity index 100% rename from scripts/generate_figure_overlays.py rename to scripts/figures/generate_figure_overlays.py diff --git a/scripts/figures/generate_xy_xz_composite_figures.py b/scripts/figures/generate_xy_xz_composite_figures.py new file mode 100644 index 000000000..757c540d2 --- /dev/null +++ b/scripts/figures/generate_xy_xz_composite_figures.py @@ -0,0 +1,612 @@ +#!/usr/bin/env python3 +""" +Generate composite figures showing XY, XZ, and YZ max projections. + +For each well, creates a figure with: +- Left panel: XY max projection composite (full height) +- Right top: XZ max projection composite +- Right bottom: YZ max projection composite + +Usage: + python scripts/figures/generate_xy_xz_composite_figures.py \ + --input-dir /path/to/max_project \ + --output-dir /path/to/output +""" + +import argparse +import csv +import logging +from dataclasses import dataclass +from pathlib import Path +import re +from typing import Dict, Iterable, Optional, List, Tuple, Union + +import matplotlib + +matplotlib.use("Agg") # Use non-interactive backend +import matplotlib.pyplot as plt +from matplotlib.gridspec import GridSpec +import numpy as np +from PIL import Image +import tifffile + +# Set up logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ColumnGroupSpec: + name: str + count: int + mode: str + + +@dataclass(frozen=True) +class MosaicTile: + well_id: str + row: int + col: int + path: Path + + +@dataclass(frozen=True) +class FigureConfig: + filename_pattern: re.Pattern + well_pattern: re.Pattern + required_projections: Tuple[str, ...] + input_extensions: Tuple[str, ...] + output_format: str + output_suffix: str + output_well_format: str + figure_size: Tuple[int, int] + dpi: int + title_format: str + panel_titles: Dict[str, str] + metadata_well_column: str + metadata_label_column: str + column_group_specs: Tuple[ColumnGroupSpec, ...] + time_label_prefix: str + mosaic_background: Tuple[int, int, int, int] + + +DEFAULT_CONFIG = FigureConfig( + filename_pattern=re.compile( + r"source_images_timepoint_(\d+)_site_(\d+)_well_(R\d+C\d+)_(XY|XZ|YZ)_max_composite", + re.IGNORECASE, + ), + well_pattern=re.compile(r"R(\d+)C(\d+)", re.IGNORECASE), + required_projections=("XY", "XZ", "YZ"), + input_extensions=(".tif", ".tiff", ".png", ".jpg", ".jpeg"), + output_format="png", + output_suffix="composite_figure", + output_well_format="rnc", + figure_size=(16, 9), + dpi=150, + title_format="Well {well_display}", + panel_titles={ + "XY": "XY Max Projection", + "XZ": "XZ Max Projection", + "YZ": "YZ Max Projection", + }, + metadata_well_column="well", + metadata_label_column="label", + column_group_specs=( + ColumnGroupSpec(name="HA-OXIME", count=5, mode="first"), + ColumnGroupSpec(name="matrigel", count=3, mode="last"), + ), + time_label_prefix="Time", + mosaic_background=(0, 0, 0, 255), +) + + +def parse_filename(filename: str, config: FigureConfig) -> Optional[Dict[str, str]]: + """Parse filename to extract well metadata.""" + match = config.filename_pattern.search(filename) + + if not match: + return None + + return { + "timepoint": match.group(1), + "site": match.group(2), + "well": match.group(3).upper(), + "projection": match.group(4).upper(), + } + + +def load_image(image_path: Path) -> Union[Image.Image, np.ndarray]: + """Load an image from disk.""" + suffix = image_path.suffix.lower() + if suffix in (".tif", ".tiff"): + data = tifffile.imread(image_path) + if data.ndim == 3 and data.shape[0] in (3, 4) and data.shape[-1] not in (3, 4): + data = np.moveaxis(data, 0, -1) + return data + return Image.open(image_path) + + +def build_title( + metadata: Dict[str, str], label: Optional[str], config: FigureConfig +) -> str: + """Build the figure title with optional label.""" + metadata_display = dict(metadata) + metadata_display["well_display"] = format_well_id(metadata["well"], config) + base = config.title_format.format(**metadata_display) + if label: + return f"{base} - {label}" + return base + + +def parse_well_position( + well_id: str, config: FigureConfig +) -> Optional[Tuple[int, int]]: + """Parse well ID into (row, col).""" + match = config.well_pattern.search(well_id) + if not match: + return None + return int(match.group(1)), int(match.group(2)) + + +def index_to_letters(index: int) -> str: + """Convert zero-based index to Excel-style letters (A, B, ..., AA).""" + index += 1 + letters = "" + while index > 0: + index, remainder = divmod(index - 1, 26) + letters = chr(ord("A") + remainder) + letters + return letters + + +def row_number_to_letters(row_number: int) -> str: + """Convert 1-based row number to letters (1 -> A).""" + if row_number < 1: + return "" + return index_to_letters(row_number - 1) + + +def letters_to_row_number(letters: str) -> int: + """Convert row letters to 1-based row number (A -> 1).""" + value = 0 + for char in letters.upper(): + if not "A" <= char <= "Z": + return 0 + value = value * 26 + (ord(char) - ord("A") + 1) + return value + + +def normalize_well_id(well_id: str, config: FigureConfig) -> str: + """Normalize well ID to R##C## format when possible.""" + cleaned = well_id.strip().upper() + if config.well_pattern.search(cleaned): + return cleaned + + match = re.match(r"^([A-Z]+)(\d+)$", cleaned) + if not match: + return cleaned + + row_letters = match.group(1) + col_number = int(match.group(2)) + row_number = letters_to_row_number(row_letters) + if row_number == 0: + return cleaned + + return f"R{row_number:02d}C{col_number:02d}" + + +def format_well_id(well_id: str, config: FigureConfig) -> str: + """Format well ID as R##C## or A01 style.""" + if config.output_well_format == "rnc": + return well_id + position = parse_well_position(well_id, config) + if not position: + return well_id + row_letters = row_number_to_letters(position[0]) + return f"{row_letters}{position[1]:02d}" + + +def slugify_label(label: str) -> str: + """Convert a label into a filename-friendly slug.""" + slug = re.sub(r"[^A-Za-z0-9_-]+", "_", label.strip()) + return slug.strip("_") + + +def build_column_group_map(columns: List[int], config: FigureConfig) -> Dict[int, str]: + """Assign labels to columns based on group specs.""" + column_labels: Dict[int, str] = {} + remaining = sorted(columns) + + for spec in config.column_group_specs: + if not remaining: + break + + if spec.mode == "first": + group_columns = remaining[: spec.count] + remaining = remaining[spec.count :] + elif spec.mode == "last": + group_columns = remaining[-spec.count :] + remaining = remaining[: -spec.count] + else: + logger.warning(f"Unknown column group mode: {spec.mode}") + continue + + for index, column in enumerate(group_columns): + time_label = index_to_letters(index) + column_labels[column] = ( + f"{spec.name} {config.time_label_prefix} {time_label}" + ) + + return column_labels + + +def build_column_labels(wells: Dict[str, dict], config: FigureConfig) -> Dict[str, str]: + """Create well -> label mapping from column grouping rules.""" + columns = [] + for payload in wells.values(): + well_id = payload["metadata"]["well"] + position = parse_well_position(well_id, config) + if position: + columns.append(position[1]) + + column_map = build_column_group_map(sorted(set(columns)), config) + well_labels: Dict[str, str] = {} + for payload in wells.values(): + well_id = payload["metadata"]["well"] + position = parse_well_position(well_id, config) + if not position: + continue + label = column_map.get(position[1]) + if label: + well_labels[well_id] = label + + return well_labels + + +def create_composite_figure( + xy_path: Path, + xz_path: Path, + yz_path: Path, + output_path: Path, + metadata: Dict[str, str], + label: Optional[str], + config: FigureConfig, +) -> None: + """Create a figure with XY, XZ, and YZ projections in a 2-column layout.""" + xy_img = load_image(xy_path) + xz_img = load_image(xz_path) + yz_img = load_image(yz_path) + + fig = plt.figure(figsize=config.figure_size) + grid = GridSpec(2, 2, figure=fig, width_ratios=[1, 1], height_ratios=[1, 1]) + + ax_xy = fig.add_subplot(grid[:, 0]) + ax_xz = fig.add_subplot(grid[0, 1]) + ax_yz = fig.add_subplot(grid[1, 1]) + + ax_xy.imshow(xy_img) + ax_xy.set_title(config.panel_titles.get("XY", "XY"), fontsize=14, weight="bold") + ax_xy.axis("off") + + ax_xz.imshow(xz_img) + ax_xz.set_title(config.panel_titles.get("XZ", "XZ"), fontsize=14, weight="bold") + ax_xz.axis("off") + + ax_yz.imshow(yz_img) + ax_yz.set_title(config.panel_titles.get("YZ", "YZ"), fontsize=14, weight="bold") + ax_yz.axis("off") + + fig.suptitle(build_title(metadata, label, config), fontsize=16, weight="bold") + + plt.tight_layout() + + output_path.parent.mkdir(parents=True, exist_ok=True) + plt.savefig(output_path, bbox_inches="tight", dpi=config.dpi) + plt.close(fig) + + +def discover_images(input_dir: Path, config: FigureConfig) -> Iterable[Path]: + """Find all candidate composite images in the input directory.""" + for ext in config.input_extensions: + yield from input_dir.glob(f"*{ext}") + + +def build_well_key(metadata: Dict[str, str]) -> str: + """Create a stable key for grouping by well/site/timepoint.""" + return ( + f"{metadata['well']}_site_{metadata['site']}_timepoint_{metadata['timepoint']}" + ) + + +def group_by_well(images: Iterable[Path], config: FigureConfig) -> Dict[str, dict]: + """Group projection images by well and projection type.""" + wells: Dict[str, dict] = {} + for img_path in images: + metadata = parse_filename(img_path.name, config) + if not metadata: + logger.warning(f"Could not parse filename: {img_path.name}") + continue + + well_key = build_well_key(metadata) + wells.setdefault(well_key, {"metadata": metadata, "projections": {}}) + wells[well_key]["projections"][metadata["projection"]] = img_path + + return wells + + +def filter_complete_wells( + wells: Dict[str, dict], config: FigureConfig +) -> Dict[str, dict]: + """Filter wells that have all required projections.""" + complete = {} + for well_key, payload in wells.items(): + projections = payload["projections"] + missing = [p for p in config.required_projections if p not in projections] + if missing: + logger.warning(f"Missing projection for {well_key}: {missing}") + continue + complete[well_key] = payload + return complete + + +def read_metadata_mapping( + metadata_csv: Optional[Path], config: FigureConfig +) -> Dict[str, str]: + """Read a mapping of well -> label from a CSV file.""" + if not metadata_csv: + return {} + + if not metadata_csv.exists(): + logger.error(f"Metadata CSV not found: {metadata_csv}") + return {} + + mapping: Dict[str, str] = {} + with metadata_csv.open("r", newline="") as csvfile: + reader = csv.DictReader(csvfile) + if not reader.fieldnames: + logger.error(f"Metadata CSV has no header: {metadata_csv}") + return {} + + for row in reader: + well = row.get(config.metadata_well_column) + label = row.get(config.metadata_label_column) + if not well or not label: + continue + normalized = normalize_well_id(well, config) + mapping[normalized] = label.strip() + + return mapping + + +def build_output_path( + output_dir: Path, + metadata: Dict[str, str], + label: Optional[str], + config: FigureConfig, +) -> Path: + """Build output filepath for a well figure.""" + well_display = format_well_id(metadata["well"], config) + stem_parts = [] + if label: + label_slug = slugify_label(label) + if label_slug: + stem_parts.append(label_slug) + stem_parts.append( + f"{well_display}_site_{metadata['site']}_timepoint_{metadata['timepoint']}" + ) + stem_parts.append(config.output_suffix) + filename = "_".join(stem_parts) + f".{config.output_format}" + return output_dir / filename + + +def normalize_mosaic_output(output_path: Path, config: FigureConfig) -> Path: + """Ensure mosaic output path has a file extension.""" + if output_path.exists() and output_path.is_dir(): + return output_path / f"mosaic.{config.output_format}" + if output_path.suffix: + return output_path + return output_path.with_suffix(f".{config.output_format}") + + +def create_mosaic( + tiles: List[MosaicTile], output_path: Path, config: FigureConfig +) -> None: + """Create a tiled mosaic image aligned by well position.""" + if not tiles: + logger.warning("No tiles available for mosaic") + return + + rows = sorted({tile.row for tile in tiles}) + cols = sorted({tile.col for tile in tiles}) + row_index = {row: idx for idx, row in enumerate(rows)} + col_index = {col: idx for idx, col in enumerate(cols)} + + images: Dict[Tuple[int, int], Image.Image] = {} + max_width = 0 + max_height = 0 + for tile in tiles: + image = Image.open(tile.path).convert("RGBA") + images[(tile.row, tile.col)] = image + max_width = max(max_width, image.width) + max_height = max(max_height, image.height) + + canvas = Image.new( + "RGBA", + (max_width * len(cols), max_height * len(rows)), + config.mosaic_background, + ) + + used_positions = set() + for tile in tiles: + key = (tile.row, tile.col) + if key in used_positions: + logger.warning( + "Duplicate tile position for row %s col %s; skipping %s", + tile.row, + tile.col, + tile.path.name, + ) + continue + used_positions.add(key) + + image = images[key] + x = col_index[tile.col] * max_width + (max_width - image.width) // 2 + y = row_index[tile.row] * max_height + (max_height - image.height) // 2 + canvas.alpha_composite(image, (x, y)) + + output_path = normalize_mosaic_output(output_path, config) + output_path.parent.mkdir(parents=True, exist_ok=True) + canvas.save(output_path) + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Generate composite figures with XY, XZ, and YZ max projections" + ) + parser.add_argument( + "--input-dir", + type=Path, + required=True, + help="Directory containing XY/XZ/YZ max projection composite images", + ) + parser.add_argument( + "--output-dir", + type=Path, + required=True, + help="Directory to save output figures", + ) + parser.add_argument( + "--output-format", + type=str, + default=DEFAULT_CONFIG.output_format, + choices=("png", "tif", "tiff"), + help="Output image format", + ) + parser.add_argument( + "--well-format", + type=str, + default=DEFAULT_CONFIG.output_well_format, + choices=("rnc", "a01"), + help="Well notation for titles/filenames (rnc or a01)", + ) + parser.add_argument( + "--metadata-csv", + type=Path, + default=None, + help="Optional CSV mapping well -> label", + ) + parser.add_argument( + "--metadata-well-column", + type=str, + default=DEFAULT_CONFIG.metadata_well_column, + help="Column name for well IDs in metadata CSV", + ) + parser.add_argument( + "--metadata-label-column", + type=str, + default=DEFAULT_CONFIG.metadata_label_column, + help="Column name for labels in metadata CSV", + ) + parser.add_argument( + "--disable-column-labels", + action="store_true", + help="Disable automatic column-based labels", + ) + parser.add_argument( + "--mosaic-output", + type=Path, + default=None, + help="Optional output path for tiled mosaic image", + ) + + args = parser.parse_args() + + if not args.input_dir.exists(): + logger.error(f"Input directory not found: {args.input_dir}") + return + + config = FigureConfig( + filename_pattern=DEFAULT_CONFIG.filename_pattern, + well_pattern=DEFAULT_CONFIG.well_pattern, + required_projections=DEFAULT_CONFIG.required_projections, + input_extensions=DEFAULT_CONFIG.input_extensions, + output_format=args.output_format, + output_suffix=DEFAULT_CONFIG.output_suffix, + output_well_format=args.well_format, + figure_size=DEFAULT_CONFIG.figure_size, + dpi=DEFAULT_CONFIG.dpi, + title_format=DEFAULT_CONFIG.title_format, + panel_titles=DEFAULT_CONFIG.panel_titles, + metadata_well_column=args.metadata_well_column, + metadata_label_column=args.metadata_label_column, + column_group_specs=DEFAULT_CONFIG.column_group_specs, + time_label_prefix=DEFAULT_CONFIG.time_label_prefix, + mosaic_background=DEFAULT_CONFIG.mosaic_background, + ) + + composite_images = list(discover_images(args.input_dir, config)) + logger.info(f"Found {len(composite_images)} composite images") + + wells = group_by_well(composite_images, config) + logger.info(f"Grouped into {len(wells)} wells") + + complete_wells = filter_complete_wells(wells, config) + logger.info(f"Found {len(complete_wells)} wells with all projections") + + label_mapping = read_metadata_mapping(args.metadata_csv, config) + column_label_mapping = {} + if not args.disable_column_labels: + column_label_mapping = build_column_labels(complete_wells, config) + + created = 0 + tiles: List[MosaicTile] = [] + for well_key, payload in sorted(complete_wells.items()): + projections = payload["projections"] + metadata = payload["metadata"] + title_label = label_mapping.get(metadata["well"]) or column_label_mapping.get( + metadata["well"] + ) + filename_label = column_label_mapping.get( + metadata["well"] + ) or label_mapping.get(metadata["well"]) + + output_path = build_output_path( + args.output_dir, metadata, filename_label, config + ) + + create_composite_figure( + projections["XY"], + projections["XZ"], + projections["YZ"], + output_path, + metadata, + title_label, + config, + ) + + position = parse_well_position(metadata["well"], config) + if args.mosaic_output and position: + tiles.append( + MosaicTile( + well_id=metadata["well"], + row=position[0], + col=position[1], + path=output_path, + ) + ) + + created += 1 + logger.info(f"✓ Created: {output_path.name}") + + if args.mosaic_output: + create_mosaic(tiles, args.mosaic_output, config) + mosaic_path = normalize_mosaic_output(args.mosaic_output, config) + logger.info(f"✓ Created mosaic: {mosaic_path.name}") + + logger.info(f"\n✅ Generated {created} composite figures in {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_xy_xz_composite_figures.py b/scripts/generate_xy_xz_composite_figures.py deleted file mode 100644 index 7b38c379d..000000000 --- a/scripts/generate_xy_xz_composite_figures.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate composite figures showing XY and XZ max projections side by side. - -For each well, creates a figure with: -- Left panel: XY max projection composite -- Right panel: XZ max projection composite - -Usage: - python scripts/generate_xy_xz_composite_figures.py \ - --input-dir /path/to/max_project \ - --output-dir /path/to/output -""" - -import argparse -import logging -from pathlib import Path -import re - -import matplotlib -matplotlib.use('Agg') # Use non-interactive backend -import matplotlib.pyplot as plt -from PIL import Image - -# Set up logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger(__name__) - - -def parse_filename(filename: str) -> dict: - """ - Parse filename to extract well and other metadata. - - Example: source_images_well_R07C05_site_1_timepoint_1_XY_max_composite.png - Returns: {'well': 'R07C05', 'site': '1', 'timepoint': '1', 'projection': 'XY'} - """ - pattern = r'well_([^_]+)_site_(\d+)_timepoint_(\d+)_(XY|XZ)_max_composite' - match = re.search(pattern, filename) - - if not match: - return None - - return { - 'well': match.group(1), - 'site': match.group(2), - 'timepoint': match.group(3), - 'projection': match.group(4) - } - - -def create_composite_figure(xy_path: Path, xz_path: Path, output_path: Path, well_id: str) -> None: - """ - Create a figure with XY and XZ projections side by side. - - Args: - xy_path: Path to XY max projection composite image - xz_path: Path to XZ max projection composite image - output_path: Path to save output figure - well_id: Well identifier for title - """ - # Load images - xy_img = Image.open(xy_path) - xz_img = Image.open(xz_path) - - # Create figure with 2 subplots side by side - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 8)) - - # Display XY projection - ax1.imshow(xy_img) - ax1.set_title('XY Max Projection', fontsize=14, weight='bold') - ax1.axis('off') - - # Display XZ projection - ax2.imshow(xz_img) - ax2.set_title('XZ Max Projection', fontsize=14, weight='bold') - ax2.axis('off') - - # Add overall title with well ID - fig.suptitle(f'Well {well_id}', fontsize=16, weight='bold') - - # Adjust layout - plt.tight_layout() - - # Save figure - output_path.parent.mkdir(parents=True, exist_ok=True) - plt.savefig(output_path, bbox_inches='tight', dpi=150) - plt.close(fig) - - -def main(): - """Main entry point.""" - parser = argparse.ArgumentParser( - description="Generate composite figures with XY and XZ max projections side by side" - ) - parser.add_argument( - '--input-dir', - type=Path, - required=True, - help='Directory containing XY and XZ max projection composite images' - ) - parser.add_argument( - '--output-dir', - type=Path, - required=True, - help='Directory to save output figures' - ) - - args = parser.parse_args() - - if not args.input_dir.exists(): - logger.error(f"Input directory not found: {args.input_dir}") - return - - # Find all composite images - composite_images = list(args.input_dir.glob('*_composite.png')) - logger.info(f"Found {len(composite_images)} composite images") - - # Group by well - wells = {} - for img_path in composite_images: - metadata = parse_filename(img_path.name) - if not metadata: - logger.warning(f"Could not parse filename: {img_path.name}") - continue - - well_key = f"{metadata['well']}_site_{metadata['site']}_timepoint_{metadata['timepoint']}" - - if well_key not in wells: - wells[well_key] = {} - - wells[well_key][metadata['projection']] = img_path - - logger.info(f"Grouped into {len(wells)} wells") - - # Create composite figures - created = 0 - for well_key, projections in sorted(wells.items()): - if 'XY' not in projections or 'XZ' not in projections: - logger.warning(f"Missing projection for {well_key}: {list(projections.keys())}") - continue - - output_filename = f"{well_key}_composite_figure.png" - output_path = args.output_dir / output_filename - - create_composite_figure( - projections['XY'], - projections['XZ'], - output_path, - well_key - ) - - created += 1 - logger.info(f"✓ Created: {output_filename}") - - logger.info(f"\n✅ Generated {created} composite figures in {args.output_dir}") - - -if __name__ == "__main__": - main() - diff --git a/scripts/napari-plugins/README.md b/scripts/napari-plugins/README.md new file mode 100644 index 000000000..58a7d0528 --- /dev/null +++ b/scripts/napari-plugins/README.md @@ -0,0 +1,120 @@ +# Napari Plugins + +This directory contains custom napari plugins for extended functionality beyond the core OpenHCS napari integration. + +## Plugins + +### `napari_orthogonal_projections.py` + +A napari plugin for creating orthogonal projections and exporting layers with advanced features. + +**Features:** + +- **Orthogonal Projections**: Create XZ, YZ, and XY max-intensity projections from 3D image stacks + - XZ: Side view (project along Y axis) + - YZ: Side view (project along X axis) + - XY: Top-down view (project along Z axis) + +- **Z-Scaling**: Adjust Z dimension scaling in projections to match physical spacing + +- **Layer Export**: Export napari layers to various formats: + - TIFF (preserves data quality) + - PNG/JPG (web-friendly formats) + - Optional dtype conversion (uint8, uint16, float32, float64) + - Apply layer scaling for proper aspect ratios + +- **Multi-channel Composites**: Create RGB composite images by merging multiple channels + - Automatic channel grouping (detects _ch1, _ch2, _channel_1 patterns) + - Respects individual channel colormaps and opacity + - Additive blending for realistic visualization + +**Dependencies:** +``` +napari +magicgui +numpy +scipy +scikit-image +tifffile +``` + +**Installation:** + +1. **As a napari plugin** (recommended): + ```bash + # Create plugin directory if needed + mkdir -p ~/.config/napari/plugins + + # Copy plugin + cp scripts/napari-plugins/napari_orthogonal_projections.py ~/.config/napari/plugins/ + + # Restart napari - widgets will appear in Plugins menu + napari + ``` + +2. **As a standalone script**: + ```bash + # Run after opening napari with data + python scripts/napari-plugins/napari_orthogonal_projections.py + ``` + +**Usage:** + +Once installed as a plugin, two widgets will appear in napari's Plugins menu: + +1. **Ortho Projections Widget**: + - Select which projections to create (XZ, YZ, XY) + - Set Z scaling factor (e.g., 5.0 for typical microscopy data) + - Toggle Z axis flipping + +2. **Export Layers Widget**: + - Choose output directory + - Select file format (TIFF, PNG, JPG) + - Choose dtype conversion + - Toggle scaling application + - Option to create multi-channel composites + - Option to export only projection layers + +**Workflow Example:** + +```python +import napari +import numpy as np + +# Create a 3D image stack +stack = np.random.rand(50, 512, 512) + +# Open napari +viewer = napari.Viewer() +viewer.add_image(stack, name='sample_stack', scale=(0.5, 1.0, 1.0)) + +# Use the plugin: +# 1. Click "Plugins > Ortho Projections" +# 2. Enable XZ and YZ projections +# 3. Set Z scale to match your data (e.g., 5.0) +# 4. Click "Create Orthogonal Projections" +# 5. Use "Plugins > Export Layers" to save results +``` + +**Key Functions:** + +- `create_projection()`: Create max projection along specified axis +- `process_layer()`: Process a single napari layer to create projection +- `export_layer()`: Export a layer to file with optional scaling/dtype conversion +- `export_composite()`: Create and export RGB composite of multiple channels +- `group_layers_by_base_name()`: Auto-group layers for compositing + +**Integration with OpenHCS:** + +This plugin is particularly useful with OpenHCS napari streaming: + +1. Use OpenHCS napari visualizer to stream real-time data +2. Apply orthogonal projections to analyze 3D structure from different angles +3. Export projections with proper scaling for publication or analysis + +**Tips:** + +- Z-scale should match the physical Z-spacing ratio (e.g., if Z-steps are 5x larger than XY pixels) +- Use TIFF for maximum quality, PNG/JPG for smaller files +- Enable "Create Composites" for multi-channel data to visualize all channels together +- "Projections Only" filter is useful when you have many layers but only want to export projections diff --git a/scripts/napari-plugins/napari_orthogonal_projections.py b/scripts/napari-plugins/napari_orthogonal_projections.py new file mode 100644 index 000000000..eda2c72c4 --- /dev/null +++ b/scripts/napari-plugins/napari_orthogonal_projections.py @@ -0,0 +1,685 @@ +#!/usr/bin/env python3 +""" +Napari plugin for creating orthogonal projections (XZ, YZ, XY) and exporting layers. + +This plugin provides GUI widgets for: +1. Creating max-intensity projections along orthogonal axes +2. Exporting layers to TIFF/PNG/JPG with optional scaling and dtype conversion +3. Creating composite RGB images from multiple channels + +Usage: + # As a napari plugin (automatic): + napari # Plugin will auto-load when placed in ~/.config/napari/plugins/ + + # As a standalone script: + python napari_orthogonal_projections.py + +Features: +- XZ projections: Side view (project along Y axis) +- YZ projections: Side view (project along X axis) +- XY projections: Top-down view (project along Z axis) +- Z-scaling: Adjust Z dimension scaling in projections +- Export to TIFF/PNG/JPG with dtype conversion +- Composite multi-channel images to RGB +""" + +from magicgui import magicgui +import napari +import numpy as np +from typing import Literal +from pathlib import Path +import tifffile +from skimage import io +from scipy.ndimage import zoom + +# ============================================================================ +# Core projection functions +# ============================================================================ + + +def create_projection(data: np.ndarray, axis: int, flip_z: bool = True) -> np.ndarray: + """ + Create a max projection along a specified axis. + + Parameters + ---------- + data : np.ndarray + 3D array with shape (Z, Y, X) + axis : int + Axis along which to project (1=Y, 2=X, 0=Z) + flip_z : bool + Whether to flip Z axis in result + + Returns + ------- + np.ndarray + 2D max projection + """ + projection = data.max(axis=axis) + + # Flip Z axis if it's first dimension in result + if flip_z and axis != 0: + projection = np.flip(projection, axis=0) + + return projection + + +def get_projection_scale( + original_scale: tuple, axis: int, z_scale_factor: float = 1.0 +) -> tuple: + """ + Calculate scale for projection based on which axis was projected. + + Parameters + ---------- + original_scale : tuple + Original (Z, Y, X) scale + axis : int + Axis that was projected out + z_scale_factor : float + Additional scaling factor for Z dimension + + Returns + ------- + tuple + Scale for 2D projection + """ + if len(original_scale) != 3: + return None + + # Apply Z scaling factor + scales = list(original_scale) + scales[0] = scales[0] * z_scale_factor + + # Remove projected axis from scale + scales.pop(axis) + return tuple(scales) + + +def add_projection_layer( + viewer: napari.Viewer, + data: np.ndarray, + name: str, + source_layer: napari.layers.Image, + scale: tuple = None, +) -> napari.layers.Image: + """ + Add a projection as a new image layer. + + Parameters + ---------- + viewer : napari.Viewer + data : np.ndarray + Projection data + name : str + Layer name + source_layer : napari.layers.Image + Original layer to copy properties from + scale : tuple, optional + Scale for projection + + Returns + ------- + napari.layers.Image + The created layer + """ + return viewer.add_image( + data, + name=name, + colormap=source_layer.colormap.name, + contrast_limits=source_layer.contrast_limits, + scale=scale, + blending=source_layer.blending, + opacity=source_layer.opacity, + ) + + +# ============================================================================ +# Layer processing +# ============================================================================ + + +def process_layer( + viewer: napari.Viewer, + layer: napari.layers.Image, + projection_type: Literal["XZ", "YZ", "XY"], + flip_z: bool = True, + z_scale_factor: float = 1.0, +) -> napari.layers.Image: + """ + Create a single projection for a layer. + + Parameters + ---------- + viewer : napari.Viewer + layer : napari.layers.Image + Source layer + projection_type : str + One of 'XZ', 'YZ', 'XY' + flip_z : bool + Whether to flip Z axis + z_scale_factor : float + Scaling factor for Z dimension + + Returns + ------- + napari.layers.Image + Created projection layer + """ + data = layer.data + + # Map projection type to axis + axis_map = { + "XZ": 1, # project along Y + "YZ": 2, # project along X + "XY": 0, # project along Z + } + + axis = axis_map[projection_type] + + # Create projection + projection = create_projection(data, axis, flip_z and projection_type != "XY") + + # Calculate scale with Z scaling factor + scale = get_projection_scale(layer.scale, axis, z_scale_factor) + + # Add layer + return add_projection_layer( + viewer, projection, f"{layer.name}_{projection_type}_max", layer, scale + ) + + +def get_3d_layers(viewer: napari.Viewer) -> list: + """Get all 3D image layers from viewer.""" + return [ + layer + for layer in viewer.layers + if hasattr(layer, "data") and layer.data.ndim == 3 + ] + + +# ============================================================================ +# Export functions +# ============================================================================ + + +def convert_dtype(data: np.ndarray, target_dtype: str) -> np.ndarray: + """ + Convert data to target dtype with appropriate scaling. + + Parameters + ---------- + data : np.ndarray + Input data + target_dtype : str + Target dtype ('as_is', 'uint8', 'uint16', 'float32', 'float64') + + Returns + ------- + np.ndarray + Converted data + """ + if target_dtype == "as_is": + return data + + # Map string to numpy dtype + dtype_map = { + "uint8": np.uint8, + "uint16": np.uint16, + "float32": np.float32, + "float64": np.float64, + } + + target = dtype_map[target_dtype] + + # If converting to integer types, normalize and scale + if target in [np.uint8, np.uint16]: + data_min = data.min() + data_max = data.max() + + if data_max > data_min: + # Normalize to 0-1 + data_norm = (data - data_min) / (data_max - data_min) + + # Scale to target range + if target == np.uint8: + return (data_norm * 255).astype(np.uint8) + elif target == np.uint16: + return (data_norm * 65535).astype(np.uint16) + else: + # Constant image + return np.zeros_like(data, dtype=target) + + # For float types, just convert + return data.astype(target) + + +def export_layer( + layer: napari.layers.Image, + output_dir: Path, + format: str = "tiff", + dtype: str = "as_is", + apply_scale: bool = True, +) -> None: + """ + Export a single layer to file. + + Parameters + ---------- + layer : napari.layers.Image + output_dir : Path + Directory to save to + format : str + File format ('tiff', 'png', 'jpg') + dtype : str + Data type conversion ('as_is', 'uint8', 'uint16', 'float32', 'float64') + apply_scale : bool + Apply layer scale to resample image (stretches Z if scaled) + """ + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Sanitize filename + safe_name = layer.name.replace("/", "_").replace("\\", "_") + + data = layer.data + + # Apply scale by resampling if requested + if apply_scale and hasattr(layer, "scale") and layer.scale is not None: + scale = layer.scale + if len(scale) == data.ndim and not all(s == 1.0 for s in scale): + # Resample with zoom + print(f"Resampling {layer.name} with scale {scale}") + data = zoom(data, scale, order=1) # order=1 is bilinear interpolation + + # Convert dtype if requested + data = convert_dtype(data, dtype) + + if format == "tiff": + filepath = output_dir / f"{safe_name}.tif" + tifffile.imwrite(filepath, data) + elif format in ["png", "jpg"]: + # PNG/JPG require uint8 + if data.dtype != np.uint8: + data = convert_dtype(data, "uint8") + filepath = output_dir / f"{safe_name}.{format}" + io.imsave(filepath, data) + + print(f"Saved: {filepath} (dtype: {data.dtype}, shape: {data.shape})") + + +def apply_colormap_to_data( + data: np.ndarray, colormap, contrast_limits: tuple = None +) -> np.ndarray: + """ + Apply a colormap to grayscale data to create RGB. + + Parameters + ---------- + data : np.ndarray + Grayscale image data + colormap : napari.utils.Colormap + Napari colormap object + contrast_limits : tuple, optional + (min, max) for contrast adjustment + + Returns + ------- + np.ndarray + RGB image with shape (..., 3) + """ + # Normalize data + if contrast_limits is not None: + vmin, vmax = contrast_limits + else: + vmin, vmax = data.min(), data.max() + + if vmax > vmin: + data_norm = np.clip((data - vmin) / (vmax - vmin), 0, 1) + else: + data_norm = np.zeros_like(data, dtype=float) + + # Use napari's colormap.map() method for proper interpolation + rgb = colormap.map(data_norm.ravel())[:, :3] # Get RGB, drop alpha + rgb = rgb.reshape(data.shape + (3,)) + + return rgb + + +def group_layers_by_base_name(layers: list) -> dict: + """ + Group layers by their base name (removing channel identifiers). + + Parameters + ---------- + layers : list + List of napari layers + + Returns + ------- + dict + Dictionary mapping base names to lists of layers + """ + import re + + groups = {} + + for layer in layers: + # Try to extract base name by removing channel identifiers + # Patterns: _channel_1, _channel_2, _ch1, _ch2, _c1, _c2, etc. + name = layer.name + + # Remove common channel patterns + base_name = re.sub(r"_channel_\d+", "", name) + base_name = re.sub(r"_ch\d+", "", base_name) + base_name = re.sub(r"_c\d+", "", base_name) + + if base_name not in groups: + groups[base_name] = [] + groups[base_name].append(layer) + + # Only return groups with multiple channels + return {k: v for k, v in groups.items() if len(v) > 1} + + +def composite_channels(layers: list) -> np.ndarray: + """ + Composite multiple channel layers into a single RGB image. + + Parameters + ---------- + layers : list + List of layers to composite (should have same shape) + + Returns + ------- + np.ndarray + Composited RGB image + """ + if not layers: + return None + + # Check that all layers have the same shape + shape = layers[0].data.shape + for layer in layers[1:]: + if layer.data.shape != shape: + print(f"Warning: Shape mismatch in composite - {layer.name}") + return None + + # Initialize RGB output + composite = np.zeros(shape + (3,), dtype=float) + + # Add each channel with its colormap + for layer in layers: + rgb = apply_colormap_to_data( + layer.data, + layer.colormap, # Pass colormap object, not name + layer.contrast_limits, + ) + + # Additive blending + composite += rgb * layer.opacity + + # Clip to valid range + composite = np.clip(composite, 0, 1) + + return composite + + +def export_composite( + layers: list, + output_dir: Path, + base_name: str, + format: str = "tiff", + dtype: str = "uint8", + apply_scale: bool = True, +) -> None: + """ + Export a composite of multiple channels. + + Parameters + ---------- + layers : list + List of channel layers to composite + output_dir : Path + Output directory + base_name : str + Base name for output file + format : str + Output format + dtype : str + Data type for export + apply_scale : bool + Whether to apply layer scaling + """ + # Apply scaling to all layers if requested + if apply_scale: + scaled_layers = [] + for layer in layers: + data = layer.data + if hasattr(layer, "scale") and layer.scale is not None: + scale = layer.scale + if len(scale) == data.ndim and not all(s == 1.0 for s in scale): + data = zoom(data, scale, order=1) + + # Create temporary layer-like object with scaled data + class ScaledLayer: + pass + + scaled_layer = ScaledLayer() + scaled_layer.data = data + scaled_layer.colormap = layer.colormap + scaled_layer.contrast_limits = layer.contrast_limits + scaled_layer.opacity = layer.opacity + scaled_layers.append(scaled_layer) + + layers = scaled_layers + + # Create composite + composite = composite_channels(layers) + + if composite is None: + print(f"Failed to create composite for {base_name}") + return + + # Convert dtype + if dtype == "uint8": + data = (composite * 255).astype(np.uint8) + elif dtype == "uint16": + data = (composite * 65535).astype(np.uint16) + else: + data = composite.astype(np.float32) + + # Save + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + safe_name = base_name.replace("/", "_").replace("\\", "_") + + if format == "tiff": + filepath = output_dir / f"{safe_name}_composite.tif" + tifffile.imwrite(filepath, data) + elif format in ["png", "jpg"]: + if data.dtype != np.uint8: + data = (composite * 255).astype(np.uint8) + filepath = output_dir / f"{safe_name}_composite.{format}" + io.imsave(filepath, data) + + print(f"Saved composite: {filepath} (shape: {data.shape})") + + +# ============================================================================ +# GUI +# ============================================================================ + + +@magicgui( + call_button="Create Orthogonal Projections", + z_scale={"min": 0.1, "max": 20.0, "step": 0.1}, +) +def create_ortho_projections( + viewer: napari.Viewer, + create_xz: bool = True, + create_yz: bool = True, + create_xy: bool = False, + flip_z: bool = True, + z_scale: float = 5.0, +) -> None: + """ + Create XZ and YZ max projections for all 3D layers. + + Parameters + ---------- + viewer : napari.Viewer + create_xz : bool + Create XZ side view projection + create_yz : bool + Create YZ side view projection + create_xy : bool + Create XY top-down projection + flip_z : bool + Flip Z axis so index 0 is at bottom + z_scale : float + Z dimension scaling factor (increases Z spacing in projections) + """ + + layers = get_3d_layers(viewer) + + if not layers: + print("No 3D layers found!") + return + + created = 0 + for layer in layers: + if create_xz: + process_layer(viewer, layer, "XZ", flip_z, z_scale) + created += 1 + + if create_yz: + process_layer(viewer, layer, "YZ", flip_z, z_scale) + created += 1 + + if create_xy: + process_layer(viewer, layer, "XY", flip_z, z_scale) + created += 1 + + print(f"Created {created} projections from {len(layers)} layers") + + +@magicgui( + call_button="Export Layers", + output_dir={"mode": "d"}, + format={"choices": ["tiff", "png", "jpg"]}, + dtype={"choices": ["as_is", "uint8", "uint16", "float32", "float64"]}, +) +def export_layers( + viewer: napari.Viewer, + output_dir: Path = Path.home(), + format: str = "tiff", + dtype: str = "as_is", + apply_scale: bool = True, + create_composites: bool = False, + projections_only: bool = True, +) -> None: + """ + Export layers to files. + + Parameters + ---------- + viewer : napari.Viewer + output_dir : Path + Directory to save images + format : str + Image format (tiff, png, jpg) + dtype : str + Convert to data type (as_is keeps original, uint8/uint16 for smaller files) + apply_scale : bool + Resample image to match visual scaling (stretches Z dimension) + create_composites : bool + Create RGB composite images by merging channels + projections_only : bool + Only export projection layers (names containing _max) + """ + + print("\n=== Export starting ===") + print(f"Create composites: {create_composites}") + print(f"Projections only: {projections_only}") + + layers_to_export = [] + + for layer in viewer.layers: + if not hasattr(layer, "data"): + continue + + if projections_only: + if "_max" in layer.name: + layers_to_export.append(layer) + else: + layers_to_export.append(layer) + + print(f"Found {len(layers_to_export)} layers to export") + for layer in layers_to_export: + print(f" - {layer.name}") + + if not layers_to_export: + print("No layers to export!") + return + + # Export individual layers + if not create_composites: + print("\nExporting individual layers...") + for layer in layers_to_export: + try: + export_layer(layer, output_dir, format, dtype, apply_scale) + except Exception as e: + print(f"ERROR: Failed to export {layer.name}: {e}") + import traceback + + traceback.print_exc() + + # Create and export composites + if create_composites: + print("\nGrouping layers for composites...") + groups = group_layers_by_base_name(layers_to_export) + + print(f"Found {len(groups)} groups:") + for base_name, group_layers in groups.items(): + print(f" Group '{base_name}':") + for layer in group_layers: + print(f" - {layer.name}") + + if not groups: + print("No multi-channel groups found for compositing!") + print("Exporting individual layers instead...") + for layer in layers_to_export: + try: + export_layer(layer, output_dir, format, dtype, apply_scale) + except Exception as e: + print(f"ERROR: Failed to export {layer.name}: {e}") + import traceback + + traceback.print_exc() + else: + for base_name, group_layers in groups.items(): + try: + print(f"\nCreating composite for '{base_name}'...") + export_composite( + group_layers, output_dir, base_name, format, dtype, apply_scale + ) + except Exception as e: + print(f"ERROR: Failed to create composite for {base_name}: {e}") + import traceback + + traceback.print_exc() + + print(f"\n=== Export complete to {output_dir} ===") + + +# ============================================================================ +# Auto-load +# ============================================================================ + +if __name__ == "__main__": + viewer = napari.current_viewer() + viewer.window.add_dock_widget(create_ortho_projections, name="Ortho Projections") + viewer.window.add_dock_widget(export_layers, name="Export Layers") diff --git a/setup.py b/setup.py index fd9636e00..646fe9336 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ "zmqruntime>=0.1.0", "pycodify>=0.1.0", "objectstate>=0.1.0", - "python-introspect>=0.1.0", + "python-introspect>=0.1.1", "metaclass-registry>=0.1.0", "arraybridge>=0.1.0", "polystore>=0.1.0", diff --git a/tests/integration/test_main.py b/tests/integration/test_main.py index 65d0ad21a..3d3354517 100644 --- a/tests/integration/test_main.py +++ b/tests/integration/test_main.py @@ -7,6 +7,7 @@ - Converted to modern Python patterns with dataclasses - Reduced verbosity and defensive programming patterns """ + import json import os import pytest @@ -17,11 +18,24 @@ from openhcs.constants.constants import VariableComponents, SequentialComponents from openhcs.constants.input_source import InputSource from openhcs.core.config import ( - GlobalPipelineConfig, MaterializationBackend, - PathPlanningConfig, StepWellFilterConfig, VFSConfig, ZarrConfig, - NapariVariableSizeHandling + GlobalPipelineConfig, + MaterializationBackend, + PathPlanningConfig, + StepWellFilterConfig, + VFSConfig, + Backend, + ZarrConfig, + NapariVariableSizeHandling, +) +from openhcs.core.config import ( + LazyStepMaterializationConfig, + LazyNapariStreamingConfig, + LazyFijiStreamingConfig, + LazyStepWellFilterConfig, + LazyPathPlanningConfig, + LazyProcessingConfig, + LazySequentialProcessingConfig, ) -from openhcs.core.config import LazyStepMaterializationConfig, LazyNapariStreamingConfig, LazyFijiStreamingConfig, LazyStepWellFilterConfig, LazyPathPlanningConfig, LazyProcessingConfig, LazySequentialProcessingConfig from openhcs.core.orchestrator.gpu_scheduler import setup_global_gpu_registry from openhcs.core.orchestrator.orchestrator import PipelineOrchestrator from openhcs.core.pipeline import Pipeline @@ -29,18 +43,32 @@ # Processing functions from openhcs.processing.backends.assemblers.assemble_stack_cpu import assemble_stack_cpu -from openhcs.processing.backends.pos_gen.ashlar_main_cpu import ashlar_compute_tile_positions_cpu +from openhcs.processing.backends.pos_gen.ashlar_main_cpu import ( + ashlar_compute_tile_positions_cpu, +) from openhcs.processing.backends.processors.numpy_processor import ( - create_composite, create_projection, stack_percentile_normalize + create_composite, + create_projection, + stack_percentile_normalize, +) +from openhcs.processing.backends.analysis.cell_counting_cpu import ( + count_cells_single_channel, + DetectionMethod, ) -from openhcs.processing.backends.analysis.cell_counting_cpu import count_cells_single_channel, DetectionMethod from openhcs.core.memory import DtypeConversion +from openhcs.core.config import DtypeConfig # Test utilities and fixtures from tests.integration.helpers.fixture_utils import ( - backend_config, base_test_dir, data_type_config, execution_mode, - microscope_config, plate_dir, test_params, print_thread_activity_report, - zmq_execution_mode + backend_config, + base_test_dir, + data_type_config, + execution_mode, + microscope_config, + plate_dir, + test_params, + print_thread_activity_report, + zmq_execution_mode, ) from openhcs.config_framework.lazy_factory import ensure_global_config_context @@ -71,20 +99,22 @@ class TestConstants: SUBDIRECTORIES_FIELD: str = "subdirectories" MIN_METADATA_ENTRIES: int = 1 - - # Required metadata fields REQUIRED_FIELDS: List[str] = None def __post_init__(self): # Use object.__setattr__ for frozen dataclass - object.__setattr__(self, 'REQUIRED_FIELDS', - ["image_files", "available_backends", "microscope_handler_name"]) + object.__setattr__( + self, + "REQUIRED_FIELDS", + ["image_files", "available_backends", "microscope_handler_name"], + ) @dataclass class TestConfig: """Configuration for test execution.""" + plate_dir: Union[Path, int] # Path for disk-based, int (plate_id) for OMERO backend_config: str execution_mode: str @@ -116,19 +146,21 @@ def _gpu_available() -> bool: - cupy device count """ # Respect visibility first - cuda_vis = os.getenv('CUDA_VISIBLE_DEVICES') - if cuda_vis is not None and (cuda_vis == '' or cuda_vis == '-1'): + cuda_vis = os.getenv("CUDA_VISIBLE_DEVICES") + if cuda_vis is not None and (cuda_vis == "" or cuda_vis == "-1"): return False # Torch try: import torch # type: ignore - if hasattr(torch, 'cuda') and torch.cuda.is_available(): + + if hasattr(torch, "cuda") and torch.cuda.is_available(): return True except Exception: pass # CuPy try: import cupy # type: ignore + try: return cupy.cuda.runtime.getDeviceCount() > 0 except Exception: @@ -137,10 +169,10 @@ def _gpu_available() -> bool: pass return False -# Auto-enable CPU-only if no GPU is available (and not explicitly forced on) -if os.getenv('OPENHCS_CPU_ONLY', 'false').lower() != 'true' and not _gpu_available(): - os.environ['OPENHCS_CPU_ONLY'] = 'true' +# Auto-enable CPU-only if no GPU is available (and not explicitly forced on) +if os.getenv("OPENHCS_CPU_ONLY", "false").lower() != "true" and not _gpu_available(): + os.environ["OPENHCS_CPU_ONLY"] = "true" def _headless_mode() -> bool: @@ -150,9 +182,9 @@ def _headless_mode() -> bool: Only CI or explicit OPENHCS_HEADLESS flag triggers headless mode. """ try: - if os.getenv('CI', '').lower() == 'true': + if os.getenv("CI", "").lower() == "true": return True - if os.getenv('OPENHCS_HEADLESS', '').lower() == 'true': + if os.getenv("OPENHCS_HEADLESS", "").lower() == "true": return True except Exception: pass @@ -163,24 +195,30 @@ def _napari_enabled() -> bool: """Check if Napari streaming should be enabled.""" if _headless_mode(): return False - return os.getenv('OPENHCS_DISABLE_NAPARI', 'false').lower() != 'true' + return os.getenv("OPENHCS_DISABLE_NAPARI", "false").lower() != "true" def _fiji_enabled() -> bool: """Check if Fiji streaming should be enabled.""" if _headless_mode(): return False - return os.getenv('OPENHCS_DISABLE_FIJI', 'false').lower() != 'true' + return os.getenv("OPENHCS_DISABLE_FIJI", "false").lower() != "true" + @pytest.fixture def test_function_dir(base_test_dir, microscope_config, request): """Create test directory for a specific test function.""" - test_name = request.node.originalname or request.node.name.split('[')[0] + test_name = request.node.originalname or request.node.name.split("[")[0] test_dir = base_test_dir / f"{test_name}[{microscope_config['format']}]" test_dir.mkdir(parents=True, exist_ok=True) yield test_dir -def create_test_pipeline(enable_napari: bool = False, enable_fiji: bool = False, sequential_config: dict = None) -> Pipeline: + +def create_test_pipeline( + enable_napari: bool = False, + enable_fiji: bool = False, + sequential_config: dict = None, +) -> Pipeline: """Create test pipeline with materialization configuration. Args: @@ -194,84 +232,115 @@ def create_test_pipeline(enable_napari: bool = False, enable_fiji: bool = False, Note: sequential_config is NOT used in this function - it should be set in PipelineConfig instead. This parameter is kept for backward compatibility but is ignored. """ - cpu_only_mode = os.getenv('OPENHCS_CPU_ONLY', 'false').lower() == 'true' + cpu_only_mode = os.getenv("OPENHCS_CPU_ONLY", "false").lower() == "true" if cpu_only_mode: position_func = ashlar_compute_tile_positions_cpu else: try: - from openhcs.processing.backends.pos_gen.ashlar_main_gpu import ashlar_compute_tile_positions_gpu + from openhcs.processing.backends.pos_gen.ashlar_main_gpu import ( + ashlar_compute_tile_positions_gpu, + ) + position_func = ashlar_compute_tile_positions_gpu except Exception: # Fallback cleanly to CPU if GPU path is unavailable - os.environ['OPENHCS_CPU_ONLY'] = 'true' + os.environ["OPENHCS_CPU_ONLY"] = "true" position_func = ashlar_compute_tile_positions_cpu return Pipeline( steps=[ Step( name="Image Enhancement Processing", - func=[(stack_percentile_normalize, {'low_percentile': 0.5, 'high_percentile': 99.5})], - step_well_filter_config=LazyStepWellFilterConfig(well_filter=CONSTANTS.STEP_WELL_FILTER_TEST), + func=[ + ( + stack_percentile_normalize, + {"low_percentile": 0.5, "high_percentile": 99.5}, + ) + ], + step_well_filter_config=LazyStepWellFilterConfig( + well_filter=CONSTANTS.STEP_WELL_FILTER_TEST + ), step_materialization_config=LazyStepMaterializationConfig(), - napari_streaming_config=LazyNapariStreamingConfig(port=5555) if enable_napari else None, - fiji_streaming_config=LazyFijiStreamingConfig() if enable_fiji else None + napari_streaming_config=LazyNapariStreamingConfig( + port=5555, enabled=enable_napari + ), + fiji_streaming_config=LazyFijiStreamingConfig(enabled=enable_fiji), ), Step( func=create_composite, processing_config=LazyProcessingConfig( variable_components=[VariableComponents.CHANNEL] ), - napari_streaming_config=LazyNapariStreamingConfig(port=5557) if enable_napari else None, - fiji_streaming_config=LazyFijiStreamingConfig(port=5556) if enable_fiji else None + napari_streaming_config=LazyNapariStreamingConfig( + port=5557, enabled=enable_napari + ), + fiji_streaming_config=LazyFijiStreamingConfig( + port=5556, enabled=enable_fiji + ), ), Step( name="Z-Stack Flattening", - func=(create_projection, {'method': 'max_projection'}), - processing_config=LazyProcessingConfig(variable_components=[VariableComponents.Z_INDEX]), - step_materialization_config=LazyStepMaterializationConfig() + func=(create_projection, {"method": "max_projection"}), + processing_config=LazyProcessingConfig( + variable_components=[VariableComponents.Z_INDEX] + ), + step_materialization_config=LazyStepMaterializationConfig(), ), Step(name="Position Computation", func=position_func), Step( name="Secondary Enhancement", - func=[(stack_percentile_normalize, {'low_percentile': 0.5, 'high_percentile': 99.5})], - processing_config=LazyProcessingConfig(input_source=InputSource.PIPELINE_START), + func=[ + ( + stack_percentile_normalize, + {"low_percentile": 0.5, "high_percentile": 99.5}, + ) + ], + processing_config=LazyProcessingConfig( + input_source=InputSource.PIPELINE_START + ), ), Step(name="CPU Assembly", func=assemble_stack_cpu), Step( name="Z-Stack Flattening", - func=(create_projection, {'method': 'max_projection'}), - processing_config=LazyProcessingConfig(variable_components=[VariableComponents.Z_INDEX]), + func=(create_projection, {"method": "max_projection"}), + processing_config=LazyProcessingConfig( + variable_components=[VariableComponents.Z_INDEX] + ), ), - Step(name="Cell Counting", - func=({'1': - ( - count_cells_single_channel, { - 'min_cell_area': 40, - 'max_cell_area': 200, - 'enable_preprocessing': False, - 'detection_method': DetectionMethod.WATERSHED, - 'dtype_conversion': DtypeConversion.UINT8, - 'return_segmentation_mask': True - } - ), - '2': - ( - count_cells_single_channel, { - 'min_cell_area': 40, - 'max_cell_area': 200, - 'enable_preprocessing': False, - 'detection_method': DetectionMethod.WATERSHED, - 'dtype_conversion': DtypeConversion.UINT8, - 'return_segmentation_mask': True - } - ) + Step( + name="Cell Counting", + func=( + { + "1": ( + count_cells_single_channel, + { + "min_cell_area": 40, + "max_cell_area": 200, + "enable_preprocessing": False, + "detection_method": DetectionMethod.WATERSHED, + "dtype_config": DtypeConfig(default_dtype_conversion=DtypeConversion.UINT8), + "return_segmentation_mask": True, + }, + ), + "2": ( + count_cells_single_channel, + { + "min_cell_area": 40, + "max_cell_area": 200, + "enable_preprocessing": False, + "detection_method": DetectionMethod.WATERSHED, + "dtype_config": DtypeConfig(default_dtype_conversion=DtypeConversion.UINT8), + "return_segmentation_mask": True, + }, + ), } ), napari_streaming_config=LazyNapariStreamingConfig( port=5559, - variable_size_handling=NapariVariableSizeHandling.PAD_TO_MAX - ) if enable_napari else None, - fiji_streaming_config=LazyFijiStreamingConfig() if enable_fiji else None + variable_size_handling=NapariVariableSizeHandling.PAD_TO_MAX, + enabled=enable_napari, + ), + fiji_streaming_config=LazyFijiStreamingConfig(enabled=enable_fiji), ), ], name=f"Multi-Subdirectory Test Pipeline{' (CPU-Only)' if cpu_only_mode else ''}", @@ -284,14 +353,16 @@ def _load_metadata(output_dir: Path) -> Dict: if not metadata_file.exists(): raise FileNotFoundError(f"Metadata file not found: {metadata_file}") - with open(metadata_file, 'r') as f: + with open(metadata_file, "r") as f: return json.load(f) def _validate_metadata_structure(metadata: Dict) -> List[str]: """Validate metadata structure and return subdirectory list.""" if CONSTANTS.SUBDIRECTORIES_FIELD not in metadata: - raise ValueError(f"Missing '{CONSTANTS.SUBDIRECTORIES_FIELD}' field in metadata") + raise ValueError( + f"Missing '{CONSTANTS.SUBDIRECTORIES_FIELD}' field in metadata" + ) subdirs = list(metadata[CONSTANTS.SUBDIRECTORIES_FIELD].keys()) @@ -307,6 +378,7 @@ def _validate_metadata_structure(metadata: Dict) -> List[str]: def _get_materialization_subdir() -> str: """Get the actual subdirectory name used by LazyStepMaterializationConfig.""" from openhcs.core.config import StepMaterializationConfig + return StepMaterializationConfig().sub_dir @@ -314,16 +386,22 @@ def _validate_subdirectory_fields(metadata: Dict) -> None: """Validate required fields in each subdirectory metadata.""" materialization_subdir = _get_materialization_subdir() - for subdir_name, subdir_metadata in metadata[CONSTANTS.SUBDIRECTORIES_FIELD].items(): + for subdir_name, subdir_metadata in metadata[ + CONSTANTS.SUBDIRECTORIES_FIELD + ].items(): missing_fields = [ - field for field in CONSTANTS.REQUIRED_FIELDS - if field not in subdir_metadata + field for field in CONSTANTS.REQUIRED_FIELDS if field not in subdir_metadata ] if missing_fields: - raise ValueError(f"Subdirectory '{subdir_name}' missing fields: {missing_fields}") + raise ValueError( + f"Subdirectory '{subdir_name}' missing fields: {missing_fields}" + ) # Validate image_files (allow empty for materialization subdirectory) - if not subdir_metadata.get("image_files") and subdir_name != materialization_subdir: + if ( + not subdir_metadata.get("image_files") + and subdir_name != materialization_subdir + ): raise ValueError(f"Subdirectory '{subdir_name}' has empty image_files list") @@ -334,15 +412,18 @@ def validate_separate_materialization(plate_dir: Path) -> None: if not (output_dir.exists() and output_dir.is_dir()): raise FileNotFoundError(f"Output directory not found: {output_dir}") - print(f"{CONSTANTS.VALIDATION_INDICATOR} Validating materialization in: {output_dir}") + print( + f"{CONSTANTS.VALIDATION_INDICATOR} Validating materialization in: {output_dir}" + ) metadata = _load_metadata(output_dir) subdirs = _validate_metadata_structure(metadata) _validate_subdirectory_fields(metadata) print(f"{CONSTANTS.VALIDATION_INDICATOR} Subdirectories: {sorted(subdirs)}") - print(f"{CONSTANTS.SUCCESS_CHECK} Materialization validation successful: {len(subdirs)} entries") - + print( + f"{CONSTANTS.SUCCESS_CHECK} Materialization validation successful: {len(subdirs)} entries" + ) def _create_pipeline_config(test_config: TestConfig) -> GlobalPipelineConfig: @@ -350,12 +431,16 @@ def _create_pipeline_config(test_config: TestConfig) -> GlobalPipelineConfig: from openhcs.constants import Microscope # Set microscope type from config - microscope = Microscope[test_config.microscope_type.upper()] if test_config.microscope_type != "auto" else Microscope.AUTO + microscope = ( + Microscope[test_config.microscope_type.upper()] + if test_config.microscope_type != "auto" + else Microscope.AUTO + ) # For OMERO tests, use omero_local backend for materialization # For other tests, use the configured backend if test_config.is_omero: - materialization_backend = MaterializationBackend('omero_local') + materialization_backend = MaterializationBackend("omero_local") else: materialization_backend = MaterializationBackend(test_config.backend_config) @@ -363,19 +448,23 @@ def _create_pipeline_config(test_config: TestConfig) -> GlobalPipelineConfig: num_workers=CONSTANTS.DEFAULT_WORKERS, microscope=microscope, path_planning_config=PathPlanningConfig( - sub_dir=CONSTANTS.DEFAULT_SUB_DIR, - output_dir_suffix=CONSTANTS.OUTPUT_SUFFIX + sub_dir=CONSTANTS.DEFAULT_SUB_DIR, output_dir_suffix=CONSTANTS.OUTPUT_SUFFIX ), vfs_config=VFSConfig(materialization_backend=materialization_backend), zarr_config=ZarrConfig(), - step_well_filter_config=StepWellFilterConfig(well_filter=CONSTANTS.GLOBAL_STEP_WELL_FILTER_TEST), - use_threading=test_config.use_threading + step_well_filter_config=StepWellFilterConfig( + well_filter=CONSTANTS.GLOBAL_STEP_WELL_FILTER_TEST + ), + use_threading=test_config.use_threading, ) -def _initialize_orchestrator(test_config: TestConfig, sequential_config=None) -> PipelineOrchestrator: +def _initialize_orchestrator( + test_config: TestConfig, sequential_config=None +) -> PipelineOrchestrator: """Initialize and configure the pipeline orchestrator.""" from polystore.base import reset_memory_backend + reset_memory_backend() setup_global_gpu_registry() @@ -402,7 +491,7 @@ def _initialize_orchestrator(test_config: TestConfig, sequential_config=None) -> # Register OMERO backend with connection in global storage registry omero_backend = OMEROLocalBackend(omero_conn=omero_manager.conn) - storage_registry['omero_local'] = omero_backend + storage_registry["omero_local"] = omero_backend # Convert sequential component names to SequentialComponents enum sequential_components = [] @@ -410,19 +499,26 @@ def _initialize_orchestrator(test_config: TestConfig, sequential_config=None) -> for comp_name in sequential_config["sequential_components"]: sequential_components.append(SequentialComponents[comp_name]) + # Determine materialization backend for OMERO tests + # For OMERO tests, use omero_local backend for materialization + # For other tests, use the configured backend + if test_config.is_omero: + materialization_backend = MaterializationBackend("omero_local") + else: + materialization_backend = MaterializationBackend(test_config.backend_config) + # Create PipelineConfig with lazy configs for proper hierarchical inheritance - # CRITICAL: Explicitly set vfs_config=None to inherit from global config - # Without this, PipelineConfig auto-creates a VFSConfig with default values (materialization_backend=DISK) - # which overrides the global config's omero_local backend for OMERO tests pipeline_config = PipelineConfig( path_planning_config=LazyPathPlanningConfig( output_dir_suffix=CONSTANTS.OUTPUT_SUFFIX ), - step_well_filter_config=LazyStepWellFilterConfig(well_filter=CONSTANTS.PIPELINE_STEP_WELL_FILTER_TEST), + step_well_filter_config=LazyStepWellFilterConfig( + well_filter=CONSTANTS.PIPELINE_STEP_WELL_FILTER_TEST + ), sequential_processing_config=LazySequentialProcessingConfig( - sequential_components=sequential_components - ) if sequential_components else None, - vfs_config=None, # Inherit from global config + sequential_components=sequential_components if sequential_components else [] + ), + vfs_config=VFSConfig(materialization_backend=materialization_backend), ) # Convert plate_dir to Path - for OMERO, format as /omero/plate_{id} @@ -455,104 +551,157 @@ def _export_pipeline_to_file(pipeline: Pipeline, plate_dir: Path) -> None: # Wrap in a complete script with header and main block lines = [] - lines.append('#!/usr/bin/env python3') + lines.append("#!/usr/bin/env python3") lines.append('"""') - lines.append(f'OpenHCS Pipeline Script - {pipeline.name}') - lines.append(f'Generated: {datetime.now()}') + lines.append(f"OpenHCS Pipeline Script - {pipeline.name}") + lines.append(f"Generated: {datetime.now()}") lines.append('"""') - lines.append('') - lines.append('from openhcs.core.pipeline import Pipeline') - lines.append('') - lines.append('') - lines.append('def create_pipeline():') + lines.append("") + lines.append("from openhcs.core.pipeline import Pipeline") + lines.append("") + lines.append("") + lines.append("def create_pipeline():") lines.append(' """Create and return the pipeline."""') - lines.append('') + lines.append("") # Indent the generated pipeline steps code - for line in python_code.split('\n'): - if line.strip() and not line.startswith('#'): - lines.append(f' {line}') - elif line.strip().startswith('#'): - lines.append(f' {line}') - - lines.append('') - lines.append(f' return Pipeline(') - lines.append(f' steps=pipeline_steps,') - lines.append(f' name={repr(pipeline.name)}') - lines.append(f' )') - lines.append('') - lines.append('') - lines.append('pipeline_steps = create_pipeline()') - lines.append('') + for line in python_code.split("\n"): + if line.strip() and not line.startswith("#"): + lines.append(f" {line}") + elif line.strip().startswith("#"): + lines.append(f" {line}") + + lines.append("") + lines.append(f" return Pipeline(") + lines.append(f" steps=pipeline_steps,") + lines.append(f" name={repr(pipeline.name)}") + lines.append(f" )") + lines.append("") + lines.append("") + lines.append("pipeline_steps = create_pipeline()") + lines.append("") # Write to file output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w') as f: - f.write('\n'.join(lines)) + with open(output_path, "w") as f: + f.write("\n".join(lines)) print(f"📝 Pipeline exported to: {output_path}") -def _execute_pipeline_phases(orchestrator: PipelineOrchestrator, pipeline: Pipeline) -> Dict: +def _drain_progress_queue(q): + """Consumer thread that drains the progress queue to prevent pipe buffer deadlock. + + Without a consumer, worker processes' feeder threads block when the pipe + buffer fills up. This prevents workers from exiting, which causes + ProcessPoolExecutor.shutdown(wait=True) to deadlock. + """ + while True: + item = q.get() # blocking get + if item is None: # sentinel to stop + break + + +def _execute_pipeline_phases( + orchestrator: PipelineOrchestrator, pipeline: Pipeline +) -> Dict: """Execute compilation and execution phases of the pipeline (direct mode).""" + import multiprocessing + import threading from openhcs.constants import MULTIPROCESSING_AXIS - from openhcs.core.orchestrator.execution_result import ExecutionResult + from openhcs.core.progress import set_progress_queue import logging + logger = logging.getLogger(__name__) wells = orchestrator.get_component_keys(MULTIPROCESSING_AXIS) if not wells: raise RuntimeError("No wells found for processing") - # Compilation phase - compilation_result = orchestrator.compile_pipelines( - pipeline_definition=pipeline.steps, - well_filter=wells - ) + mp_ctx = multiprocessing.get_context("spawn") + progress_queue = mp_ctx.Queue() - # Extract compiled_contexts from the result dict - compiled_contexts = compilation_result['compiled_contexts'] - - if len(compiled_contexts) != len(wells): - raise RuntimeError(f"Compilation failed: expected {len(wells)} contexts, got {len(compiled_contexts)}") - - # DEBUG: Log Napari streaming configs - logger.info("=" * 80) - logger.info("DEBUG: Checking Napari streaming configurations") - for well_id, ctx in compiled_contexts.items(): - logger.info(f"Well {well_id}: {len(ctx.required_visualizers)} visualizers") - for i, vis_info in enumerate(ctx.required_visualizers): - config = vis_info['config'] - if hasattr(config, 'port'): - logger.info(f" Visualizer {i}: Napari port {config.port}") - else: - logger.info(f" Visualizer {i}: {type(config).__name__}") - logger.info("=" * 80) - - # Execution phase - results = orchestrator.execute_compiled_plate( - pipeline_definition=pipeline.steps, - compiled_contexts=compiled_contexts + # Start a consumer thread to drain progress messages and prevent pipe + # buffer deadlock. This mirrors the pattern used by zmq_execution_server. + consumer = threading.Thread( + target=_drain_progress_queue, args=(progress_queue,), daemon=True ) + consumer.start() - if len(results) != len(wells): - raise RuntimeError(f"Execution failed: expected {len(wells)} results, got {len(results)}") + try: + set_progress_queue(progress_queue) + compilation_result = orchestrator.compile_pipelines( + pipeline_definition=pipeline.steps, well_filter=wells + ) - # Validate all wells succeeded - failed_wells = [ - well_id for well_id, result in results.items() - if not result.is_success() - ] - if failed_wells: - raise RuntimeError(f"Wells failed execution: {failed_wells}") + # Extract compiled_contexts from the result dict + compiled_contexts = compilation_result["compiled_contexts"] + + if len(compiled_contexts) != len(wells): + raise RuntimeError( + f"Compilation failed: expected {len(wells)} contexts, got {len(compiled_contexts)}" + ) + + # DEBUG: Log Napari streaming configs + logger.info("=" * 80) + logger.info("DEBUG: Checking Napari streaming configurations") + for well_id, ctx in compiled_contexts.items(): + logger.info(f"Well {well_id}: {len(ctx.required_visualizers)} visualizers") + for i, vis_info in enumerate(ctx.required_visualizers): + config = vis_info["config"] + if hasattr(config, "port"): + logger.info(f" Visualizer {i}: Napari port {config.port}") + else: + logger.info(f" Visualizer {i}: {type(config).__name__}") + logger.info("=" * 80) + + # Execution phase - pass the progress queue + import time + progress_context = { + "execution_id": f"direct::{int(time.time() * 1_000_000)}", + "plate_id": str(orchestrator.plate_path), + "axis_id": "", + } + + results = orchestrator.execute_compiled_plate( + pipeline_definition=pipeline.steps, + compiled_contexts=compiled_contexts, + progress_queue=progress_queue, + progress_context=progress_context, + ) - return results + if len(results) != len(wells): + raise RuntimeError( + f"Execution failed: expected {len(results)} results, got {len(results)}" + ) + # Validate all wells succeeded + failed_wells = [ + well_id for well_id, result in results.items() if not result.is_success() + ] + if failed_wells: + raise RuntimeError(f"Wells failed execution: {failed_wells}") -def _execute_pipeline_zmq(test_config: TestConfig, pipeline: Pipeline, global_config: GlobalPipelineConfig, pipeline_config: PipelineConfig) -> Dict: + return results + finally: + set_progress_queue(None) + # Send sentinel to stop the consumer thread, then wait for it + progress_queue.put(None) + consumer.join(timeout=10) + progress_queue.close() + progress_queue.join_thread() + + +def _execute_pipeline_zmq( + test_config: TestConfig, + pipeline: Pipeline, + global_config: GlobalPipelineConfig, + pipeline_config: PipelineConfig, +) -> Dict: """Execute pipeline using ZMQ execution client.""" from openhcs.runtime.zmq_execution_client import ZMQExecutionClient import logging + logger = logging.getLogger(__name__) logger.info("🔌 Executing pipeline via ZMQ execution client") @@ -571,12 +720,12 @@ def _execute_pipeline_zmq(test_config: TestConfig, pipeline: Pipeline, global_co plate_id=str(test_config.plate_dir), pipeline_steps=pipeline.steps, global_config=global_config, - pipeline_config=pipeline_config + pipeline_config=pipeline_config, ) # Check response - if response.get('status') == 'error': - error_msg = response.get('message', 'Unknown error') + if response.get("status") == "error": + error_msg = response.get("message", "Unknown error") raise RuntimeError(f"ZMQ execution failed: {error_msg}") logger.info(f"✅ ZMQ execution completed: {response.get('status')}") @@ -584,7 +733,7 @@ def _execute_pipeline_zmq(test_config: TestConfig, pipeline: Pipeline, global_co # Convert response to results format expected by tests # ZMQ returns {'status': 'complete', 'execution_id': '...', 'result': {...}} # We need to return the result dict - return response.get('result', {}) + return response.get("result", {}) finally: # Cleanup @@ -592,7 +741,9 @@ def _execute_pipeline_zmq(test_config: TestConfig, pipeline: Pipeline, global_co logger.info("🔌 Disconnected from ZMQ execution server") -def _execute_pipeline_with_mode(test_config: TestConfig, pipeline: Pipeline, zmq_mode: str, sequential_config=None) -> Dict: +def _execute_pipeline_with_mode( + test_config: TestConfig, pipeline: Pipeline, zmq_mode: str, sequential_config=None +) -> Dict: """ Execute pipeline using either direct orchestrator or ZMQ client based on mode. @@ -605,6 +756,7 @@ def _execute_pipeline_with_mode(test_config: TestConfig, pipeline: Pipeline, zmq Execution results dict """ import logging + logger = logging.getLogger(__name__) if zmq_mode == "zmq": @@ -619,49 +771,92 @@ def _execute_pipeline_with_mode(test_config: TestConfig, pipeline: Pipeline, zmq for comp_name in sequential_config["sequential_components"]: sequential_components.append(SequentialComponents[comp_name]) + # Determine materialization backend for OMERO tests + # For OMERO tests, use omero_local backend for materialization + # For other tests, use the configured backend + if test_config.is_omero: + materialization_backend = MaterializationBackend("omero_local") + else: + materialization_backend = MaterializationBackend(test_config.backend_config) + # Create pipeline config with lazy configs - # NOTE: No need to set vfs_config explicitly - the compiler will automatically - # validate and correct the backend based on microscope compatibility - from openhcs.core.config import LazyPathPlanningConfig, LazyStepWellFilterConfig, LazySequentialProcessingConfig + from openhcs.core.config import ( + LazyPathPlanningConfig, + LazyStepWellFilterConfig, + LazySequentialProcessingConfig, + ) + pipeline_config = PipelineConfig( path_planning_config=LazyPathPlanningConfig( output_dir_suffix=CONSTANTS.OUTPUT_SUFFIX ), - step_well_filter_config=LazyStepWellFilterConfig(well_filter=CONSTANTS.PIPELINE_STEP_WELL_FILTER_TEST), + step_well_filter_config=LazyStepWellFilterConfig( + well_filter=CONSTANTS.PIPELINE_STEP_WELL_FILTER_TEST + ), sequential_processing_config=LazySequentialProcessingConfig( sequential_components=sequential_components - ) if sequential_components else None, + if sequential_components + else [] + ), + vfs_config=VFSConfig(materialization_backend=materialization_backend), ) - return _execute_pipeline_zmq(test_config, pipeline, global_config, pipeline_config) + return _execute_pipeline_zmq( + test_config, pipeline, global_config, pipeline_config + ) else: logger.info("🔧 Using direct orchestrator mode") orchestrator = _initialize_orchestrator(test_config, sequential_config) return _execute_pipeline_phases(orchestrator, pipeline) -def test_main(plate_dir: Union[Path, str, int], backend_config: str, data_type_config: Dict, execution_mode: str, zmq_execution_mode: str, microscope_config: Dict, enable_napari: bool, enable_fiji: bool, sequential_config: Dict): +def test_main( + plate_dir: Union[Path, str, int], + backend_config: str, + data_type_config: Dict, + execution_mode: str, + zmq_execution_mode: str, + microscope_config: Dict, + enable_napari: bool, + enable_fiji: bool, + sequential_config: Dict, +): """Unified test for all combinations of microscope types, backends, data types, execution modes, and sequential processing.""" # Handle both Path and int (OMERO plate_id) if isinstance(plate_dir, int): - test_config = TestConfig(plate_dir, backend_config, execution_mode, microscope_config) + test_config = TestConfig( + plate_dir, backend_config, execution_mode, microscope_config + ) else: - test_config = TestConfig(Path(plate_dir), backend_config, execution_mode, microscope_config) + test_config = TestConfig( + Path(plate_dir), backend_config, execution_mode, microscope_config + ) # For OMERO tests, connect to OMERO (automatically starts if needed) if test_config.is_omero: from openhcs.runtime.omero_instance_manager import OMEROInstanceManager + print("� Connecting to OMERO (will auto-start if needed)...") omero_manager = OMEROInstanceManager() - if not omero_manager.connect(timeout=120): # Increased timeout for docker startup - pytest.skip("OMERO server not available and could not be started automatically. Check Docker installation.") + if not omero_manager.connect( + timeout=120 + ): # Increased timeout for docker startup + pytest.skip( + "OMERO server not available and could not be started automatically. Check Docker installation." + ) omero_manager.close() print("✅ OMERO server is ready") - print(f"{CONSTANTS.START_INDICATOR} with plate: {plate_dir}, backend: {backend_config}, microscope: {microscope_config['format']}, mode: {execution_mode}, zmq: {zmq_execution_mode}, sequential: {sequential_config['name']}") + print( + f"{CONSTANTS.START_INDICATOR} with plate: {plate_dir}, backend: {backend_config}, microscope: {microscope_config['format']}, mode: {execution_mode}, zmq: {zmq_execution_mode}, sequential: {sequential_config['name']}" + ) # Create pipeline with sequential configuration - pipeline = create_test_pipeline(enable_napari=enable_napari, enable_fiji=enable_fiji, sequential_config=sequential_config) + pipeline = create_test_pipeline( + enable_napari=enable_napari, + enable_fiji=enable_fiji, + sequential_config=sequential_config, + ) # If this configuration should fail validation, expect ValueError during compilation if sequential_config.get("should_fail", False): @@ -671,12 +866,18 @@ def test_main(plate_dir: Union[Path, str, int], backend_config: str, data_type_c _export_pipeline_to_file(pipeline, test_config.plate_dir) # Execute using the specified mode (direct or zmq) - should fail during compilation - _execute_pipeline_with_mode(test_config, pipeline, zmq_execution_mode, sequential_config) + _execute_pipeline_with_mode( + test_config, pipeline, zmq_execution_mode, sequential_config + ) # Verify the error message contains the expected substring expected_error = sequential_config.get("expected_error", "") - assert expected_error in str(exc_info.value), f"Expected error message to contain '{expected_error}', but got: {exc_info.value}" - print(f"✅ Validation correctly rejected invalid configuration: {exc_info.value}") + assert expected_error in str(exc_info.value), ( + f"Expected error message to contain '{expected_error}', but got: {exc_info.value}" + ) + print( + f"✅ Validation correctly rejected invalid configuration: {exc_info.value}" + ) return # Test passed - invalid config was rejected as expected # Valid configuration - should succeed @@ -685,7 +886,9 @@ def test_main(plate_dir: Union[Path, str, int], backend_config: str, data_type_c _export_pipeline_to_file(pipeline, test_config.plate_dir) # Execute using the specified mode (direct or zmq) - results = _execute_pipeline_with_mode(test_config, pipeline, zmq_execution_mode, sequential_config) + results = _execute_pipeline_with_mode( + test_config, pipeline, zmq_execution_mode, sequential_config + ) # Validate materialization (skip for OMERO - different validation needed) if not test_config.is_omero: @@ -696,13 +899,13 @@ def test_main(plate_dir: Union[Path, str, int], backend_config: str, data_type_c import webbrowser # Get OMERO web URL from config - omero_web_host = os.getenv('OMERO_WEB_HOST', 'localhost') - omero_web_port = os.getenv('OMERO_WEB_PORT', '4080') + omero_web_host = os.getenv("OMERO_WEB_HOST", "localhost") + omero_web_port = os.getenv("OMERO_WEB_PORT", "4080") plate_url = f"http://{omero_web_host}:{omero_web_port}/webclient/?show=plate-{plate_dir}" - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print(f"OMERO Plate URL: {plate_url}") - print(f"{'='*80}\n") + print(f"{'=' * 80}\n") # Open browser webbrowser.open(plate_url) @@ -711,7 +914,14 @@ def test_main(plate_dir: Union[Path, str, int], backend_config: str, data_type_c print(f"{CONSTANTS.SUCCESS_INDICATOR} ({len(results)} wells processed)") -def _test_main_with_code_serialization(plate_dir: Union[Path, str, int], backend_config: str, data_type_config: Dict, execution_mode: str, zmq_execution_mode: str, microscope_config: Dict): +def _test_main_with_code_serialization( + plate_dir: Union[Path, str, int], + backend_config: str, + data_type_config: Dict, + execution_mode: str, + zmq_execution_mode: str, + microscope_config: Dict, +): """ DISABLED: Code serialization test (not run as pytest test). @@ -730,14 +940,21 @@ def _test_main_with_code_serialization(plate_dir: Union[Path, str, int], backend """ # Handle both Path and int (OMERO plate_id) if isinstance(plate_dir, int): - test_config = TestConfig(plate_dir, backend_config, execution_mode, microscope_config) + test_config = TestConfig( + plate_dir, backend_config, execution_mode, microscope_config + ) else: - test_config = TestConfig(Path(plate_dir), backend_config, execution_mode, microscope_config) + test_config = TestConfig( + Path(plate_dir), backend_config, execution_mode, microscope_config + ) - print(f"{CONSTANTS.START_INDICATOR} [CODE SERIALIZATION TEST] with plate: {plate_dir}, backend: {backend_config}, mode: {execution_mode}, zmq: {zmq_execution_mode}") + print( + f"{CONSTANTS.START_INDICATOR} [CODE SERIALIZATION TEST] with plate: {plate_dir}, backend: {backend_config}, mode: {execution_mode}, zmq: {zmq_execution_mode}" + ) # Step 1: Create objects normally from polystore.base import reset_memory_backend + reset_memory_backend() setup_global_gpu_registry() @@ -748,7 +965,9 @@ def _test_main_with_code_serialization(plate_dir: Union[Path, str, int], backend path_planning_config=LazyPathPlanningConfig( output_dir_suffix=CONSTANTS.OUTPUT_SUFFIX ), - step_well_filter_config=LazyStepWellFilterConfig(well_filter=CONSTANTS.PIPELINE_STEP_WELL_FILTER_TEST), + step_well_filter_config=LazyStepWellFilterConfig( + well_filter=CONSTANTS.PIPELINE_STEP_WELL_FILTER_TEST + ), ) pipeline = create_test_pipeline() @@ -805,19 +1024,21 @@ def _test_main_with_code_serialization(plate_dir: Union[Path, str, int], backend # Recreate GlobalPipelineConfig global_config_namespace = {} exec(global_config_code, global_config_namespace) - recreated_global_config = global_config_namespace['config'] + recreated_global_config = global_config_namespace["config"] # Recreate PipelineConfig pipeline_config_namespace = {} exec(pipeline_config_code, pipeline_config_namespace) - recreated_pipeline_config = pipeline_config_namespace['config'] + recreated_pipeline_config = pipeline_config_namespace["config"] # Recreate Pipeline steps pipeline_steps_namespace = {} exec(pipeline_steps_code, pipeline_steps_namespace) - recreated_pipeline_steps = pipeline_steps_namespace['pipeline_steps'] + recreated_pipeline_steps = pipeline_steps_namespace["pipeline_steps"] - print(f" - Recreated GlobalPipelineConfig: {type(recreated_global_config).__name__}") + print( + f" - Recreated GlobalPipelineConfig: {type(recreated_global_config).__name__}" + ) print(f" - Recreated PipelineConfig: {type(recreated_pipeline_config).__name__}") print(f" - Recreated Pipeline steps: {len(recreated_pipeline_steps)} steps") @@ -835,7 +1056,9 @@ def _test_main_with_code_serialization(plate_dir: Union[Path, str, int], backend # Check Pipeline steps assert len(recreated_pipeline_steps) == len(pipeline.steps) - for i, (orig_step, recreated_step) in enumerate(zip(pipeline.steps, recreated_pipeline_steps)): + for i, (orig_step, recreated_step) in enumerate( + zip(pipeline.steps, recreated_pipeline_steps) + ): assert type(orig_step) == type(recreated_step) assert orig_step.name == recreated_step.name print(f" ✅ Pipeline steps match ({len(recreated_pipeline_steps)} steps)") @@ -844,25 +1067,33 @@ def _test_main_with_code_serialization(plate_dir: Union[Path, str, int], backend print("\n🚀 Step 5: Executing pipeline with recreated objects...") # Create Pipeline object from recreated steps - recreated_pipeline = Pipeline( - steps=recreated_pipeline_steps, - name=pipeline.name - ) + recreated_pipeline = Pipeline(steps=recreated_pipeline_steps, name=pipeline.name) # Execute using the specified mode (direct or zmq) if zmq_execution_mode == "zmq": # For ZMQ mode, use the recreated configs directly - results = _execute_pipeline_zmq(test_config, recreated_pipeline, recreated_global_config, recreated_pipeline_config) + results = _execute_pipeline_zmq( + test_config, + recreated_pipeline, + recreated_global_config, + recreated_pipeline_config, + ) else: # For direct mode, set up global context and use orchestrator ensure_global_config_context(GlobalPipelineConfig, recreated_global_config) - orchestrator = PipelineOrchestrator(test_config.plate_dir, pipeline_config=recreated_pipeline_config) + orchestrator = PipelineOrchestrator( + test_config.plate_dir, pipeline_config=recreated_pipeline_config + ) orchestrator.initialize() results = _execute_pipeline_phases(orchestrator, recreated_pipeline) validate_separate_materialization(test_config.plate_dir) print_thread_activity_report() - print(f"\n{CONSTANTS.SUCCESS_INDICATOR} [CODE SERIALIZATION TEST] ({len(results)} wells processed)") + print( + f"\n{CONSTANTS.SUCCESS_INDICATOR} [CODE SERIALIZATION TEST] ({len(results)} wells processed)" + ) print("✅ Code-based serialization works perfectly!") - print(" This proves we can use Python code instead of pickling for remote execution.") + print( + " This proves we can use Python code instead of pickling for remote execution." + ) diff --git a/tests/unit/progress/test_projection.py b/tests/unit/progress/test_projection.py new file mode 100644 index 000000000..e0724231c --- /dev/null +++ b/tests/unit/progress/test_projection.py @@ -0,0 +1,167 @@ +from openhcs.core.progress import ProgressEvent, ProgressPhase, ProgressStatus +from openhcs.core.progress.projection import ( + PlateRuntimeState, + build_execution_runtime_projection, +) + + +def _event( + *, + execution_id: str = "exec-1", + plate_id: str = "/tmp/plate", + axis_id: str = "", + step_name: str = "pipeline", + phase: ProgressPhase, + status: ProgressStatus, + percent: float, + completed: int = 0, + total: int = 1, + total_wells=None, +) -> ProgressEvent: + return ProgressEvent( + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name=step_name, + phase=phase, + status=status, + percent=percent, + completed=completed, + total=total, + timestamp=1.0, + pid=1234, + total_wells=total_wells, + ) + + +def test_projection_marks_plate_compiled_when_all_known_wells_compiled(): + events = [ + _event( + phase=ProgressPhase.INIT, + status=ProgressStatus.STARTED, + percent=0.0, + total_wells=["A01", "B01"], + ), + _event( + axis_id="A01", + step_name="compilation", + phase=ProgressPhase.COMPILE, + status=ProgressStatus.SUCCESS, + percent=100.0, + ), + _event( + axis_id="B01", + step_name="compilation", + phase=ProgressPhase.COMPILE, + status=ProgressStatus.SUCCESS, + percent=100.0, + ), + ] + + projection = build_execution_runtime_projection({"exec-1": events}) + plate = projection.get_plate("/tmp/plate", "exec-1") + + assert plate is not None + assert plate.state == PlateRuntimeState.COMPILED + assert round(plate.percent, 1) == 100.0 + assert projection.compiled_count == 1 + + +def test_projection_marks_plate_executing_from_pipeline_channel(): + events = [ + _event( + phase=ProgressPhase.INIT, + status=ProgressStatus.STARTED, + percent=0.0, + total_wells=["A01", "B01"], + ), + _event( + axis_id="A01", + step_name="normalize", + phase=ProgressPhase.STEP_COMPLETED, + status=ProgressStatus.SUCCESS, + percent=50.0, + completed=1, + total=2, + ), + ] + + projection = build_execution_runtime_projection({"exec-1": events}) + plate = projection.get_plate("/tmp/plate", "exec-1") + + assert plate is not None + assert plate.state == PlateRuntimeState.EXECUTING + assert round(plate.percent, 1) == 25.0 + assert projection.executing_count == 1 + + +def test_projection_dedupes_multiple_execution_ids_for_same_plate(): + exec1 = _event( + execution_id="exec-1", + phase=ProgressPhase.COMPILE, + status=ProgressStatus.SUCCESS, + percent=100.0, + axis_id="A01", + step_name="compilation", + ) + exec2 = _event( + execution_id="exec-2", + phase=ProgressPhase.STEP_STARTED, + status=ProgressStatus.RUNNING, + percent=50.0, + axis_id="A01", + step_name="normalize", + completed=1, + total=2, + ) + # Make second execution newer. + exec2 = exec2.replace(timestamp=2.0) + + projection = build_execution_runtime_projection( + { + "exec-1": [exec1], + "exec-2": [exec2], + } + ) + + assert len(projection.plates) == 2 # raw snapshots + assert len(projection.by_plate_latest) == 1 # deduped visible plate set + assert projection.executing_count == 1 + assert projection.compiled_count == 0 + + +def test_projection_marks_plate_failed_on_axis_error(): + events = [ + _event( + phase=ProgressPhase.INIT, + status=ProgressStatus.STARTED, + percent=0.0, + total_wells=["A01", "B01"], + ), + _event( + axis_id="A01", + step_name="normalize", + phase=ProgressPhase.AXIS_ERROR, + status=ProgressStatus.ERROR, + percent=50.0, + completed=1, + total=2, + ), + _event( + axis_id="B01", + step_name="pipeline", + phase=ProgressPhase.AXIS_COMPLETED, + status=ProgressStatus.SUCCESS, + percent=100.0, + completed=2, + total=2, + ), + ] + + projection = build_execution_runtime_projection({"exec-1": events}) + plate = projection.get_plate("/tmp/plate", "exec-1") + + assert plate is not None + assert plate.state == PlateRuntimeState.FAILED + assert round(plate.percent, 1) == 75.0 + assert projection.failed_count == 1 diff --git a/tests/unit/progress/test_registry_channel_keying.py b/tests/unit/progress/test_registry_channel_keying.py new file mode 100644 index 000000000..0f30b650d --- /dev/null +++ b/tests/unit/progress/test_registry_channel_keying.py @@ -0,0 +1,57 @@ +from openhcs.core.progress import ProgressEvent, ProgressPhase, ProgressStatus, registry + + +def _event( + *, + phase: ProgressPhase, + status: ProgressStatus, + percent: float, + axis_id: str = "A01", + step_name: str = "pipeline", + completed: int = 0, + total: int = 1, +) -> ProgressEvent: + return ProgressEvent( + execution_id="exec-1", + plate_id="/tmp/plate", + axis_id=axis_id, + step_name=step_name, + phase=phase, + status=status, + percent=percent, + completed=completed, + total=total, + timestamp=1.0, + pid=1111, + ) + + +def test_registry_keeps_pipeline_and_step_channels_separate(): + tracker = registry() + tracker.clear_all() + + pipeline_event = _event( + phase=ProgressPhase.STEP_COMPLETED, + status=ProgressStatus.SUCCESS, + percent=25.0, + completed=1, + total=4, + ) + step_event = _event( + phase=ProgressPhase.PATTERN_GROUP, + status=ProgressStatus.RUNNING, + percent=50.0, + completed=1, + total=2, + step_name="max_project", + ) + + tracker.register_event("exec-1", pipeline_event) + tracker.register_event("exec-1", step_event) + + events = tracker.get_events("exec-1") + assert len(events) == 2 + phase_set = {event.phase for event in events} + assert phase_set == {ProgressPhase.STEP_COMPLETED, ProgressPhase.PATTERN_GROUP} + + tracker.clear_all() diff --git a/tests/unit/pyqt_gui/test_batch_workflow_compile_engine.py b/tests/unit/pyqt_gui/test_batch_workflow_compile_engine.py new file mode 100644 index 000000000..7d68a952d --- /dev/null +++ b/tests/unit/pyqt_gui/test_batch_workflow_compile_engine.py @@ -0,0 +1,132 @@ +import asyncio + +import pytest + +from openhcs.pyqt_gui.widgets.shared.services.batch_workflow_service import ( + BatchWorkflowService, + CompileJob, +) +from zmqruntime.execution import BatchSubmitWaitEngine + + +def _job(plate_path: str) -> CompileJob: + return CompileJob( + plate_path=plate_path, + plate_name=plate_path, + definition_pipeline=[], + pipeline_config={"x": 1}, + ) + + +def test_compile_policy_with_engine_collects_artifacts_and_callbacks(): + service = BatchWorkflowService.__new__(BatchWorkflowService) + callback_events: list[tuple[str, str, str]] = [] + + async def fake_submit_compile_job(*, job: CompileJob, zmq_client, loop) -> str: + return f"exec-{job.plate_path}" + + async def fake_wait_compile_job( + *, submission_id: str, job: CompileJob, zmq_client, loop + ) -> None: + callback_events.append(("wait", job.plate_path, submission_id)) + + service._submit_compile_job = fake_submit_compile_job + service._wait_compile_job = fake_wait_compile_job + + jobs = [_job("/tmp/a"), _job("/tmp/b")] + policy = service._make_compile_policy( + zmq_client=object(), + loop=object(), + fail_fast_submit=False, + fail_fast_wait=False, + on_wait_success=lambda job, submission_id, _idx, _total: callback_events.append( + ("success", job.plate_path, submission_id) + ), + ) + artifacts = asyncio.run(BatchSubmitWaitEngine[CompileJob]().run(jobs, policy)) + + assert artifacts == { + "/tmp/a": "exec-/tmp/a", + "/tmp/b": "exec-/tmp/b", + } + assert callback_events == [ + ("wait", "/tmp/a", "exec-/tmp/a"), + ("success", "/tmp/a", "exec-/tmp/a"), + ("wait", "/tmp/b", "exec-/tmp/b"), + ("success", "/tmp/b", "exec-/tmp/b"), + ] + + +def test_compile_policy_fail_fast_submit_raises(): + service = BatchWorkflowService.__new__(BatchWorkflowService) + + async def fake_submit_compile_job(*, job: CompileJob, zmq_client, loop) -> str: + if job.plate_path == "/tmp/b": + raise RuntimeError("boom") + return f"exec-{job.plate_path}" + + async def fake_wait_compile_job( + *, submission_id: str, job: CompileJob, zmq_client, loop + ) -> None: + return None + + service._submit_compile_job = fake_submit_compile_job + service._wait_compile_job = fake_wait_compile_job + + policy = service._make_compile_policy( + zmq_client=object(), + loop=object(), + fail_fast_submit=True, + fail_fast_wait=True, + ) + + with pytest.raises(RuntimeError, match="boom"): + asyncio.run( + BatchSubmitWaitEngine[CompileJob]().run( + [_job("/tmp/a"), _job("/tmp/b"), _job("/tmp/c")], policy + ) + ) + +def test_compile_policy_non_fail_fast_wait_tracks_error_and_finally(): + service = BatchWorkflowService.__new__(BatchWorkflowService) + callbacks = {"start": [], "success": [], "error": [], "finally": []} + + async def fake_submit_compile_job(*, job: CompileJob, zmq_client, loop) -> str: + return f"exec-{job.plate_path}" + + async def fake_wait_compile_job( + *, submission_id: str, job: CompileJob, zmq_client, loop + ) -> None: + if job.plate_path == "/tmp/b": + raise RuntimeError("compile failed") + + service._submit_compile_job = fake_submit_compile_job + service._wait_compile_job = fake_wait_compile_job + + policy = service._make_compile_policy( + zmq_client=object(), + loop=object(), + fail_fast_submit=False, + fail_fast_wait=False, + on_wait_start=lambda job, _idx, _total: callbacks["start"].append( + job.plate_path + ), + on_wait_success=lambda job, execution_id, _idx, _total: callbacks[ + "success" + ].append((job.plate_path, execution_id)), + on_wait_error=lambda job, error, _idx, _total: callbacks["error"].append( + (job.plate_path, str(error)) + ), + on_wait_finally=lambda job, _idx, _total: callbacks["finally"].append( + job.plate_path + ), + ) + artifacts = asyncio.run( + BatchSubmitWaitEngine[CompileJob]().run([_job("/tmp/a"), _job("/tmp/b")], policy) + ) + + assert artifacts == {"/tmp/a": "exec-/tmp/a"} + assert callbacks["start"] == ["/tmp/a", "/tmp/b"] + assert callbacks["success"] == [("/tmp/a", "exec-/tmp/a")] + assert callbacks["error"] == [("/tmp/b", "compile failed")] + assert callbacks["finally"] == ["/tmp/a", "/tmp/b"] diff --git a/tests/unit/pyqt_gui/test_execution_server_status_presenter.py b/tests/unit/pyqt_gui/test_execution_server_status_presenter.py new file mode 100644 index 000000000..79096115d --- /dev/null +++ b/tests/unit/pyqt_gui/test_execution_server_status_presenter.py @@ -0,0 +1,81 @@ +from openhcs.core.progress.projection import ( + ExecutionRuntimeProjection, + PlateRuntimeProjection, + PlateRuntimeState, +) +from openhcs.pyqt_gui.widgets.shared.services.execution_server_status_presenter import ( + ExecutionServerStatusPresenter, +) + + +def test_execution_server_status_presenter_returns_ready_when_no_plates(): + presenter = ExecutionServerStatusPresenter() + projection = ExecutionRuntimeProjection() + + view = presenter.build_status_text( + projection=projection, + server_info=None, + ) + + assert view.text == "Ready" + + +def test_execution_server_status_presenter_includes_projection_counts(): + presenter = ExecutionServerStatusPresenter() + plate_one = PlateRuntimeProjection( + execution_id="exec-1", + plate_id="/tmp/p1", + state=PlateRuntimeState.COMPILING, + percent=40.0, + axis_progress=tuple(), + latest_timestamp=1.0, + ) + plate_two = PlateRuntimeProjection( + execution_id="exec-2", + plate_id="/tmp/p2", + state=PlateRuntimeState.EXECUTING, + percent=85.0, + axis_progress=tuple(), + latest_timestamp=1.0, + ) + projection = ExecutionRuntimeProjection( + by_plate_latest={"/tmp/p1": plate_one, "/tmp/p2": plate_two}, + compiling_count=1, + executing_count=1, + compiled_count=0, + complete_count=0, + overall_percent=62.5, + ) + view = presenter.build_status_text( + projection=projection, + server_info=None, + ) + + assert ( + view.text + == "Server: ⏳ 1 compiling, ⚙️ 1 executing | 2 plates | avg 62.5%" + ) + + +def test_execution_server_status_presenter_includes_failed_projection_count(): + presenter = ExecutionServerStatusPresenter() + failed_plate = PlateRuntimeProjection( + execution_id="exec-3", + plate_id="/tmp/p3", + state=PlateRuntimeState.FAILED, + percent=10.0, + axis_progress=tuple(), + latest_timestamp=1.0, + ) + projection = ExecutionRuntimeProjection( + by_plate_latest={"/tmp/p3": failed_plate}, + failed_count=1, + overall_percent=10.0, + ) + + view = presenter.build_status_text( + projection=projection, + server_info=None, + ) + + assert view.text == "Server: ❌ 1 failed | 1 plates | avg 10.0%" diff --git a/tests/unit/pyqt_gui/test_execution_server_summary.py b/tests/unit/pyqt_gui/test_execution_server_summary.py new file mode 100644 index 000000000..dd79d1842 --- /dev/null +++ b/tests/unit/pyqt_gui/test_execution_server_summary.py @@ -0,0 +1,64 @@ +from openhcs.pyqt_gui.widgets.shared.server_browser import ( + ProgressNode, + summarize_execution_server, +) + + +def test_execution_server_summary_handles_empty_nodes(): + summary = summarize_execution_server([]) + assert summary.status_text == "✅ Idle" + assert summary.info_text == "" + + +def test_execution_server_summary_formats_counts_and_average(): + nodes = [ + ProgressNode( + node_id="p1", + node_type="plate", + label="p1", + status="⏳ Queued", + info="", + percent=0.0, + ), + ProgressNode( + node_id="p2", + node_type="plate", + label="p2", + status="⚙️ Executing", + info="", + percent=50.0, + ), + ProgressNode( + node_id="p3", + node_type="plate", + label="p3", + status="✅ Complete", + info="", + percent=100.0, + ), + ] + + summary = summarize_execution_server(nodes) + + assert "⏳ 1 queued" in summary.status_text + assert "⚙️ 1 executing" in summary.status_text + assert "✅ 1 complete" in summary.status_text + assert summary.info_text == "Avg: 50.0% | 3 plates" + + +def test_execution_server_summary_includes_failed_count(): + nodes = [ + ProgressNode( + node_id="p1", + node_type="plate", + label="p1", + status="❌ Failed", + info="", + percent=25.0, + ) + ] + + summary = summarize_execution_server(nodes) + + assert "❌ 1 failed" in summary.status_text + assert summary.info_text == "Avg: 25.0% | 1 plates" diff --git a/tests/unit/pyqt_gui/test_interval_snapshot_poller.py b/tests/unit/pyqt_gui/test_interval_snapshot_poller.py new file mode 100644 index 000000000..80714dc47 --- /dev/null +++ b/tests/unit/pyqt_gui/test_interval_snapshot_poller.py @@ -0,0 +1,73 @@ +import threading +import time + +from pyqt_reactive.services import ( + CallbackIntervalSnapshotPollerPolicy, + IntervalSnapshotPoller, +) + + +def _wait_until(predicate, timeout_seconds: float = 1.0) -> None: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + if predicate(): + return + time.sleep(0.01) + raise AssertionError("Timed out waiting for condition") + + +def test_interval_snapshot_poller_reports_only_changed_snapshots(): + snapshots = [{"value": 1}, {"value": 1}, {"value": 2}] + changed: list[dict[str, int]] = [] + + poller = IntervalSnapshotPoller[dict[str, int]]( + CallbackIntervalSnapshotPollerPolicy( + fetch_snapshot_fn=lambda: snapshots.pop(0), + clone_snapshot_fn=lambda snapshot: dict(snapshot), + poll_interval_seconds_value=0.0, + on_snapshot_changed_fn=lambda snapshot: changed.append(snapshot), + ) + ) + + poller.tick() + _wait_until(lambda: not poller.is_poll_inflight()) + poller.tick() + _wait_until(lambda: not poller.is_poll_inflight()) + poller.tick() + _wait_until(lambda: not poller.is_poll_inflight()) + + assert changed == [{"value": 1}, {"value": 2}] + assert poller.get_snapshot_copy() == {"value": 2} + + +def test_interval_snapshot_poller_reset_drops_stale_inflight_result(): + release_fetch = threading.Event() + changed: list[dict[str, int]] = [] + fetch_values = [{"value": 1}, {"value": 2}] + + def _fetch() -> dict[str, int]: + release_fetch.wait(timeout=1.0) + return fetch_values.pop(0) + + poller = IntervalSnapshotPoller[dict[str, int]]( + CallbackIntervalSnapshotPollerPolicy( + fetch_snapshot_fn=_fetch, + clone_snapshot_fn=lambda snapshot: dict(snapshot), + poll_interval_seconds_value=0.0, + on_snapshot_changed_fn=lambda snapshot: changed.append(snapshot), + ) + ) + + poller.tick() + _wait_until(lambda: poller.is_poll_inflight()) + poller.reset() + release_fetch.set() + _wait_until(lambda: not poller.is_poll_inflight()) + + assert poller.get_snapshot_copy() is None + assert changed == [] + + poller.tick() + _wait_until(lambda: not poller.is_poll_inflight()) + assert changed == [{"value": 2}] + assert poller.get_snapshot_copy() == {"value": 2} diff --git a/tests/unit/pyqt_gui/test_main_config_propagation.py b/tests/unit/pyqt_gui/test_main_config_propagation.py new file mode 100644 index 000000000..6da797cb8 --- /dev/null +++ b/tests/unit/pyqt_gui/test_main_config_propagation.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from openhcs.core.config import GlobalPipelineConfig +from openhcs.pyqt_gui.main import OpenHCSMainWindow + + +@dataclass +class _ConfigAwareStub: + calls: int = 0 + last_config: GlobalPipelineConfig | None = None + + def on_config_changed(self, new_config: GlobalPipelineConfig) -> None: + self.calls += 1 + self.last_config = new_config + + +@dataclass +class _ServiceAdapterStub: + calls: int = 0 + last_config: GlobalPipelineConfig | None = None + + def set_global_config(self, config: GlobalPipelineConfig) -> None: + self.calls += 1 + self.last_config = config + + +def test_on_config_changed_propagates_to_embedded_widgets() -> None: + plate_manager = _ConfigAwareStub() + pipeline_editor = _ConfigAwareStub() + service_adapter = _ServiceAdapterStub() + + main_like = type("MainLike", (), {})() + main_like.global_config = GlobalPipelineConfig() + main_like.service_adapter = service_adapter + main_like.plate_manager_widget = plate_manager + main_like.pipeline_editor_widget = pipeline_editor + main_like.floating_windows = {} + + new_config = GlobalPipelineConfig(num_workers=2) + OpenHCSMainWindow.on_config_changed(main_like, new_config) + + assert main_like.global_config == new_config + assert service_adapter.calls == 1 + assert service_adapter.last_config == new_config + + assert plate_manager.calls == 1 + assert plate_manager.last_config == new_config + + assert pipeline_editor.calls == 1 + assert pipeline_editor.last_config == new_config diff --git a/tests/unit/pyqt_gui/test_plate_status_presenter.py b/tests/unit/pyqt_gui/test_plate_status_presenter.py new file mode 100644 index 000000000..8e813cc9b --- /dev/null +++ b/tests/unit/pyqt_gui/test_plate_status_presenter.py @@ -0,0 +1,75 @@ +from openhcs.core.orchestrator.orchestrator import OrchestratorState +from openhcs.core.progress.projection import PlateRuntimeProjection, PlateRuntimeState +from openhcs.pyqt_gui.widgets.shared.services.execution_state import ( + TerminalExecutionStatus, +) +from openhcs.pyqt_gui.widgets.shared.services.plate_status_presenter import ( + PlateStatusPresenter, +) + + +def _runtime_plate(state: PlateRuntimeState, percent: float = 0.0) -> PlateRuntimeProjection: + return PlateRuntimeProjection( + execution_id="exec-1", + plate_id="/tmp/plate", + state=state, + percent=percent, + axis_progress=tuple(), + latest_timestamp=1.0, + ) + + +def test_runtime_projection_is_canonical_over_local_flags(): + prefix = PlateStatusPresenter.build_status_prefix( + orchestrator_state=OrchestratorState.EXEC_FAILED, + is_init_pending=True, + is_compile_pending=True, + is_execution_active=False, + terminal_status=TerminalExecutionStatus.FAILED, + queue_position=3, + runtime_projection=_runtime_plate(PlateRuntimeState.COMPILING, 42.0), + ) + + assert prefix == "⏳ Compiling 42.0%" + + +def test_queue_position_used_when_runtime_projection_absent(): + prefix = PlateStatusPresenter.build_status_prefix( + orchestrator_state=None, + is_init_pending=False, + is_compile_pending=False, + is_execution_active=False, + terminal_status=None, + queue_position=2, + runtime_projection=None, + ) + + assert prefix == "⏳ Queued 0.0% (q#2)" + + +def test_pending_status_when_active_without_runtime_or_queue(): + prefix = PlateStatusPresenter.build_status_prefix( + orchestrator_state=None, + is_init_pending=False, + is_compile_pending=False, + is_execution_active=True, + terminal_status=None, + queue_position=None, + runtime_projection=None, + ) + + assert prefix == "⏳ Pending" + + +def test_orchestrator_fallback_for_idle_case(): + prefix = PlateStatusPresenter.build_status_prefix( + orchestrator_state=OrchestratorState.COMPILED, + is_init_pending=False, + is_compile_pending=False, + is_execution_active=False, + terminal_status=None, + queue_position=None, + runtime_projection=None, + ) + + assert prefix == "✓ Compiled" diff --git a/tests/unit/pyqt_gui/test_progress_topology_state.py b/tests/unit/pyqt_gui/test_progress_topology_state.py new file mode 100644 index 000000000..c544fccfa --- /dev/null +++ b/tests/unit/pyqt_gui/test_progress_topology_state.py @@ -0,0 +1,66 @@ +import pytest + +from openhcs.core.progress import ProgressEvent, ProgressPhase, ProgressStatus +from openhcs.pyqt_gui.widgets.shared.server_browser import ProgressTopologyState + + +def _event( + *, + execution_id: str = "exec-1", + plate_id: str = "/tmp/plate", + axis_id: str = "A01", + phase: ProgressPhase = ProgressPhase.STEP_STARTED, + worker_slot: str | None = "worker_0", + owned_wells: list[str] | None = None, + worker_assignments: dict[str, list[str]] | None = None, + total_wells: list[str] | None = None, +) -> ProgressEvent: + return ProgressEvent( + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name="s", + phase=phase, + status=ProgressStatus.RUNNING, + percent=0.0, + completed=0, + total=1, + timestamp=1.0, + pid=1, + worker_slot=worker_slot, + owned_wells=owned_wells, + worker_assignments=worker_assignments, + total_wells=total_wells, + ) + + +def test_progress_topology_state_tracks_assignments_and_wells(): + state = ProgressTopologyState() + event = _event( + worker_assignments={"worker_0": ["A01"]}, + owned_wells=["A01"], + total_wells=["A01", "B01"], + ) + + state.register_event(event) + + key = ("exec-1", "/tmp/plate") + assert state.worker_assignments[key] == {"worker_0": ["A01"]} + assert state.known_wells[key] == ["A01", "B01"] + assert "exec-1" in state.seen_execution_ids + + +def test_progress_topology_state_rejects_claim_mismatch(): + state = ProgressTopologyState() + seed_event = _event( + worker_assignments={"worker_0": ["A01"]}, + owned_wells=["A01"], + ) + state.register_event(seed_event) + + bad_event = _event( + worker_assignments=None, + owned_wells=["B01"], + ) + with pytest.raises(ValueError, match="Worker claim mismatch"): + state.register_event(bad_event) diff --git a/tests/unit/pyqt_gui/test_progress_tree_aggregation.py b/tests/unit/pyqt_gui/test_progress_tree_aggregation.py new file mode 100644 index 000000000..8187f73a8 --- /dev/null +++ b/tests/unit/pyqt_gui/test_progress_tree_aggregation.py @@ -0,0 +1,459 @@ +from openhcs.core.progress import ProgressEvent, ProgressPhase, ProgressStatus +from openhcs.pyqt_gui.widgets.shared.zmq_server_manager import ZMQServerManagerWidget +from openhcs.pyqt_gui.widgets.shared.server_browser import ProgressTreeBuilder +from pyqt_reactive.services import DefaultServerInfoParser + + +def _event( + *, + phase: ProgressPhase, + status: ProgressStatus, + percent: float, + execution_id: str = "exec-1", + plate_id: str = "/tmp/plate", + axis_id: str = "A01", + step_name: str = "pipeline", + completed: int = 0, + total: int = 1, +) -> ProgressEvent: + return ProgressEvent( + execution_id=execution_id, + plate_id=plate_id, + axis_id=axis_id, + step_name=step_name, + phase=phase, + status=status, + percent=percent, + completed=completed, + total=total, + timestamp=1.0, + pid=1111, + ) + + +def _execution_server_info( + *, + queued: list[dict] | None = None, + running: list[dict] | None = None, + compile_status: str | None = None, +): + parser = DefaultServerInfoParser() + payload = { + "port": 7777, + "ready": True, + "server": "OpenHCSExecutionServer", + "log_file_path": "/tmp/server.log", + "workers": [], + "running_executions": running or [], + "queued_executions": queued or [], + } + if compile_status is not None: + payload["compile_status"] = compile_status + payload["compile_message"] = "" + return parser.parse(payload) + + +def test_worker_tree_uses_pipeline_percent_for_well_and_parent_aggregation( + monkeypatch, +): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._worker_assignments = {("exec-1", "/tmp/plate"): {"worker_0": ["A01"]}} + manager._known_wells = {("exec-1", "/tmp/plate"): ["A01"]} + + pipeline_event = _event( + phase=ProgressPhase.STEP_COMPLETED, + status=ProgressStatus.SUCCESS, + percent=25.0, + completed=1, + total=4, + step_name="normalize", + ) + step_event = _event( + phase=ProgressPhase.PATTERN_GROUP, + status=ProgressStatus.RUNNING, + percent=50.0, + completed=1, + total=2, + step_name="max_project", + ) + + nodes = manager._build_progress_tree({"exec-1": [pipeline_event, step_event]}) + + assert len(nodes) == 1 + plate = nodes[0] + worker = plate.children[0] + well = worker.children[0] + step = well.children[0] + + assert round(step.percent, 1) == 50.0 + assert round(well.percent, 1) == 25.0 + assert round(worker.percent, 1) == 25.0 + assert round(plate.percent, 1) == 25.0 + + +def test_compile_tree_marks_plate_as_compiled_at_100_percent(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._worker_assignments = {} + manager._known_wells = {("exec-1", "/tmp/plate"): ["A01", "B01"]} + + compile_a = _event( + phase=ProgressPhase.COMPILE, + status=ProgressStatus.SUCCESS, + percent=100.0, + axis_id="A01", + step_name="compilation", + ) + compile_b = _event( + phase=ProgressPhase.COMPILE, + status=ProgressStatus.SUCCESS, + percent=100.0, + axis_id="B01", + step_name="compilation", + ) + + nodes = manager._build_progress_tree({"exec-1": [compile_a, compile_b]}) + + assert len(nodes) == 1 + plate = nodes[0] + assert round(plate.percent, 1) == 100.0 + assert plate.status == "✅ Compiled" + + +def test_compile_tree_marks_failed_compilation(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._worker_assignments = {} + manager._known_wells = {("exec-1", "/tmp/plate"): ["A01"]} + + compile_failed = _event( + phase=ProgressPhase.COMPILE, + status=ProgressStatus.FAILED, + percent=35.0, + axis_id="A01", + step_name="compilation", + ) + + nodes = manager._build_progress_tree({"exec-1": [compile_failed]}) + + assert len(nodes) == 1 + plate = nodes[0] + assert round(plate.percent, 1) == 35.0 + assert plate.status == "❌ Compile Failed" + assert plate.children[0].status == "❌ Failed" + + +def test_worker_tree_marks_failed_well_and_plate_status(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._worker_assignments = {("exec-1", "/tmp/plate"): {"worker_0": ["A01"]}} + manager._known_wells = {("exec-1", "/tmp/plate"): ["A01"]} + + failed_event = _event( + phase=ProgressPhase.AXIS_ERROR, + status=ProgressStatus.ERROR, + percent=42.0, + completed=1, + total=2, + step_name="normalize", + ) + + nodes = manager._build_progress_tree({"exec-1": [failed_event]}) + + assert len(nodes) == 1 + plate = nodes[0] + worker = plate.children[0] + well = worker.children[0] + + assert plate.status == "❌ Failed" + assert worker.status == "❌ 1 failed" + assert well.status == "❌ Failed" + + +def test_queued_plates_are_injected_without_progress_events(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + + nodes = manager._merge_server_snapshot_nodes( + [], + _execution_server_info( + queued=[ + { + "execution_id": "exec-123", + "plate_id": "/tmp/plate_a", + "queue_position": 1, + }, + { + "execution_id": "exec-456", + "plate_id": "/tmp/plate_b", + "queue_position": 2, + }, + ] + ), + ) + + assert len(nodes) == 2 + assert all(node.status == "⏳ Queued" for node in nodes) + + +def test_queued_overrides_compiled_plate_node(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._worker_assignments = {} + manager._known_wells = {("exec-compile", "/tmp/plate_a"): ["A01"]} + + compiled_node = manager._build_progress_tree( + { + "exec-compile": [ + _event( + execution_id="exec-compile", + phase=ProgressPhase.COMPILE, + status=ProgressStatus.SUCCESS, + percent=100.0, + plate_id="/tmp/plate_a", + axis_id="A01", + step_name="compilation", + ) + ] + } + )[0] + + merged = manager._merge_server_snapshot_nodes( + [compiled_node], + _execution_server_info( + queued=[ + { + "execution_id": "exec-run", + "plate_id": "/tmp/plate_a", + "queue_position": 2, + } + ] + ), + ) + + assert len(merged) == 1 + assert merged[0].status == "⏳ Queued" + assert merged[0].percent == 0.0 + assert merged[0].info == "0.0% (q#2)" + assert merged[0].children == [] + + +def test_running_plate_is_injected_without_progress_events(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._get_plate_name = lambda plate_id, exec_id=None: ( + f"{plate_id.split('/')[-1]} ({exec_id[:8]})" + if exec_id + else plate_id.split("/")[-1] + ) + + merged = manager._merge_server_snapshot_nodes( + [], + _execution_server_info( + running=[ + { + "execution_id": "exec-run", + "plate_id": "/tmp/plate_a", + "compile_only": True, + } + ], + ), + ) + + assert len(merged) == 1 + assert merged[0].status == "⏳ Compiling" + assert merged[0].percent == 0.0 + + +def test_running_snapshot_without_progress_defaults_to_executing(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._get_plate_name = lambda plate_id, exec_id=None: plate_id.split("/")[-1] + + merged = manager._merge_server_snapshot_nodes( + [], + _execution_server_info( + running=[{"execution_id": "exec-run", "plate_id": "/tmp/plate_a"}], + compile_status="compiled success", + ), + ) + + assert len(merged) == 1 + assert merged[0].status == "⚙️ Executing" + assert merged[0].percent == 0.0 + + +def test_running_compile_only_snapshot_without_compile_status_is_compiling(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._get_plate_name = lambda plate_id, exec_id=None: plate_id.split("/")[-1] + + merged = manager._merge_server_snapshot_nodes( + [], + _execution_server_info( + running=[ + { + "execution_id": "exec-compile", + "plate_id": "/tmp/plate_a", + "compile_only": True, + } + ], + ), + ) + + assert len(merged) == 1 + assert merged[0].status == "⏳ Compiling" + assert merged[0].percent == 0.0 + + +def test_progress_client_connection_tracks_execution_server_presence(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + + class _DummyClient: + def __init__(self) -> None: + self.disconnected = False + + def is_connected(self) -> bool: + return True + + def disconnect(self) -> None: + self.disconnected = True + + calls = {"setup": 0} + manager._zmq_client = None + + def _setup() -> None: + calls["setup"] += 1 + manager._zmq_client = _DummyClient() + + manager._setup_progress_client = _setup + + manager._sync_progress_client_connection( + [_execution_server_info(running=[], queued=[])] + ) + assert calls["setup"] == 1 + assert manager._zmq_client is not None + + manager._sync_progress_client_connection( + [_execution_server_info(running=[], queued=[])] + ) + assert calls["setup"] == 1 + + client = manager._zmq_client + manager._sync_progress_client_connection([]) + assert manager._zmq_client is None + assert client.disconnected is True + + +def test_init_only_execution_with_assignments_renders_as_queued_not_compiling( + monkeypatch, +): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._worker_assignments = {("exec-run", "/tmp/plate"): {"worker_0": ["A01"]}} + manager._known_wells = {("exec-run", "/tmp/plate"): ["A01"]} + + init_event = _event( + execution_id="exec-run", + phase=ProgressPhase.INIT, + status=ProgressStatus.RUNNING, + percent=0.0, + axis_id="", + step_name="init", + ) + + nodes = manager._build_progress_tree({"exec-run": [init_event]}) + + assert len(nodes) == 1 + plate = nodes[0] + assert plate.status == "⏳ Queued" + assert plate.percent == 0.0 + # Worker shows "⚙️ Starting" because execution has started (INIT phase) + # Wells still show "⏳ Queued" until they receive individual progress events + assert plate.children[0].status == "⚙️ Starting" + assert plate.children[0].children[0].status == "⏳ Queued" + + +def test_running_snapshot_does_not_override_progress_queued_node(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._get_plate_name = lambda plate_id, exec_id=None: plate_id.split("/")[-1] + manager._worker_assignments = {("exec-run", "/tmp/plate"): {"worker_0": ["A01"]}} + manager._known_wells = {("exec-run", "/tmp/plate"): ["A01"]} + + queued_plate = manager._build_progress_tree( + { + "exec-run": [ + _event( + execution_id="exec-run", + phase=ProgressPhase.INIT, + status=ProgressStatus.RUNNING, + percent=0.0, + plate_id="/tmp/plate", + axis_id="", + step_name="init", + ) + ] + } + )[0] + assert queued_plate.status == "⏳ Queued" + + merged = manager._merge_server_snapshot_nodes( + [queued_plate], + _execution_server_info( + running=[{"execution_id": "exec-run", "plate_id": "/tmp/plate"}], + compile_status="compiling", + ), + ) + + assert len(merged) == 1 + assert merged[0].status == "⏳ Queued" + + +def test_compile_events_with_worker_assignments_stay_in_compile_mode(monkeypatch): + monkeypatch.setattr(ZMQServerManagerWidget, "__del__", lambda self: None) + manager = ZMQServerManagerWidget.__new__(ZMQServerManagerWidget) + manager._progress_tree_builder = ProgressTreeBuilder() + manager._worker_assignments = { + ("exec-compile", "/tmp/plate"): {"worker_0": ["A01"]} + } + manager._known_wells = {("exec-compile", "/tmp/plate"): ["A01"]} + + compile_event = _event( + execution_id="exec-compile", + phase=ProgressPhase.COMPILE, + status=ProgressStatus.SUCCESS, + percent=100.0, + plate_id="/tmp/plate", + axis_id="A01", + step_name="compilation", + ) + init_event = _event( + execution_id="exec-compile", + phase=ProgressPhase.INIT, + status=ProgressStatus.RUNNING, + percent=0.0, + plate_id="/tmp/plate", + axis_id="", + step_name="init", + ) + + nodes = manager._build_progress_tree({"exec-compile": [init_event, compile_event]}) + + assert len(nodes) == 1 + plate = nodes[0] + assert plate.status == "✅ Compiled" + assert plate.children[0].node_type == "compilation" diff --git a/tests/unit/pyqt_gui/test_server_kill_service.py b/tests/unit/pyqt_gui/test_server_kill_service.py new file mode 100644 index 000000000..a2d43d0d3 --- /dev/null +++ b/tests/unit/pyqt_gui/test_server_kill_service.py @@ -0,0 +1,82 @@ +from openhcs.pyqt_gui.widgets.shared.server_browser import ( + ServerKillPlan, + ServerKillService, +) + + +class _FakeRegistry: + def __init__(self): + self.removed: list[int] = [] + + def remove_tracker(self, port: int) -> None: + self.removed.append(port) + + +def test_server_kill_service_strict_failures_returns_error(): + registry = _FakeRegistry() + killed_ports: list[int] = [] + + def kill_server_fn(port: int, graceful: bool, config) -> bool: + if port == 7778: + return False + return True + + service = ServerKillService( + kill_server_fn=kill_server_fn, + queue_tracker_registry_factory=lambda: registry, + config=object(), + ) + plan = ServerKillPlan( + graceful=True, + strict_failures=True, + emit_signal_on_failure=False, + success_message="ok", + ) + + success, message = service.kill_ports( + ports=[7777, 7778], + plan=plan, + on_server_killed=lambda port: killed_ports.append(port), + log_info=lambda *_args, **_kwargs: None, + log_warning=lambda *_args, **_kwargs: None, + log_error=lambda *_args, **_kwargs: None, + ) + + assert not success + assert "7778" in message + assert killed_ports == [7777] + assert registry.removed == [7777] + + +def test_server_kill_service_emit_on_failure_marks_killed_for_refresh(): + registry = _FakeRegistry() + killed_ports: list[int] = [] + + def kill_server_fn(port: int, graceful: bool, config) -> bool: + return False + + service = ServerKillService( + kill_server_fn=kill_server_fn, + queue_tracker_registry_factory=lambda: registry, + config=object(), + ) + plan = ServerKillPlan( + graceful=False, + strict_failures=False, + emit_signal_on_failure=True, + success_message="done", + ) + + success, message = service.kill_ports( + ports=[8888, 9999], + plan=plan, + on_server_killed=lambda port: killed_ports.append(port), + log_info=lambda *_args, **_kwargs: None, + log_warning=lambda *_args, **_kwargs: None, + log_error=lambda *_args, **_kwargs: None, + ) + + assert success + assert message == "done" + assert killed_ports == [8888, 9999] + assert registry.removed == [8888, 9999] diff --git a/tests/unit/pyqt_gui/test_server_row_presenter.py b/tests/unit/pyqt_gui/test_server_row_presenter.py new file mode 100644 index 000000000..afc3d1708 --- /dev/null +++ b/tests/unit/pyqt_gui/test_server_row_presenter.py @@ -0,0 +1,77 @@ +import pytest + +try: + from PyQt6.QtWidgets import QTreeWidgetItem + + PYQT_AVAILABLE = True +except Exception: + PYQT_AVAILABLE = False + +from pyqt_reactive.services import DefaultServerInfoParser +from openhcs.pyqt_gui.widgets.shared.server_browser import ServerRowPresenter + + +@pytest.mark.skipif(not PYQT_AVAILABLE, reason="PyQt6 not available") +def test_server_row_presenter_renders_execution_server(): + parser = DefaultServerInfoParser() + info = parser.parse( + { + "port": 7777, + "ready": True, + "server": "OpenHCSExecutionServer", + "log_file_path": "/tmp/server.log", + "workers": [], + "running_executions": [], + "queued_executions": [], + } + ) + + created: list[tuple[str, str, str, dict]] = [] + presenter = ServerRowPresenter( + create_tree_item=lambda display, status, extra, data: ( + created.append((display, status, extra, data)), + QTreeWidgetItem([display, status, extra]), + )[1], + update_execution_server_item=lambda _item, _data: None, + log_warning=lambda *_args, **_kwargs: None, + ) + + item = presenter.render_server(info, "✅") + + assert item.text(0) == "Port 7777 - Execution Server" + assert item.text(1) == "✅ Idle" + assert len(created) == 1 + + +@pytest.mark.skipif(not PYQT_AVAILABLE, reason="PyQt6 not available") +def test_server_row_presenter_populates_execution_children(): + parser = DefaultServerInfoParser() + info = parser.parse( + { + "port": 7777, + "ready": True, + "server": "OpenHCSExecutionServer", + "log_file_path": "/tmp/server.log", + "workers": [], + "running_executions": [], + "queued_executions": [], + } + ) + + called = {"value": 0} + presenter = ServerRowPresenter( + create_tree_item=lambda display, status, extra, data: QTreeWidgetItem( + [display, status, extra] + ), + update_execution_server_item=lambda item, data: ( + called.__setitem__("value", called["value"] + 1), + item.addChild(QTreeWidgetItem(["child", "", ""])), + ), + log_warning=lambda *_args, **_kwargs: None, + ) + + item = QTreeWidgetItem(["server", "", ""]) + has_children = presenter.populate_server_children(info, item) + + assert called["value"] == 1 + assert has_children diff --git a/tests/unit/pyqt_gui/test_server_tree_population.py b/tests/unit/pyqt_gui/test_server_tree_population.py new file mode 100644 index 000000000..75d5d32a1 --- /dev/null +++ b/tests/unit/pyqt_gui/test_server_tree_population.py @@ -0,0 +1,58 @@ +from pyqt_reactive.services import DefaultServerInfoParser + +from openhcs.pyqt_gui.widgets.shared.server_browser import ServerTreePopulation + + +class _FakeTree: + def __init__(self): + self.items = [] + + def addTopLevelItem(self, item): + self.items.append(item) + + +def _execution_server_info(): + parser = DefaultServerInfoParser() + return parser.parse( + { + "port": 7777, + "ready": True, + "server": "OpenHCSExecutionServer", + "log_file_path": "/tmp/server.log", + "workers": [], + "running_executions": [], + "queued_executions": [], + } + ) + + +def test_server_tree_population_adds_scanned_servers(): + rendered = [] + populated = [] + tree = _FakeTree() + + population = ServerTreePopulation( + create_tree_item=lambda display, status, info, data: { + "display": display, + "status": status, + "info": info, + "data": data, + "children": [], + }, + render_server=lambda info, status_icon: ( + rendered.append((info.port, status_icon)), + {"port": info.port}, + )[1], + populate_server_children=lambda info, item: ( + populated.append((info.port, item.get("port"))), + False, + )[1], + log_info=lambda *_args, **_kwargs: None, + ) + population._get_launching_viewers = lambda: {} + + population.populate_tree(tree=tree, parsed_servers=[_execution_server_info()]) + + assert len(tree.items) == 1 + assert rendered == [(7777, "✅")] + assert populated == [(7777, 7777)] diff --git a/tests/unit/pyqt_gui/test_tree_rebuild_coordinator.py b/tests/unit/pyqt_gui/test_tree_rebuild_coordinator.py new file mode 100644 index 000000000..a4523a2af --- /dev/null +++ b/tests/unit/pyqt_gui/test_tree_rebuild_coordinator.py @@ -0,0 +1,56 @@ +import pytest + +try: + from PyQt6.QtCore import Qt + from PyQt6.QtWidgets import QApplication, QTreeWidget, QTreeWidgetItem + + PYQT_AVAILABLE = True +except Exception: + PYQT_AVAILABLE = False + +from pyqt_reactive.widgets.shared import TreeRebuildCoordinator + + +@pytest.mark.skipif(not PYQT_AVAILABLE, reason="PyQt6 not available") +def test_tree_rebuild_coordinator_preserves_selection_and_expansion(): + app = QApplication.instance() + if app is None: + app = QApplication([]) + + tree = QTreeWidget() + tree.setHeaderLabels(["Server", "Status", "Info"]) + + old_server = QTreeWidgetItem(["Port 7777", "ok", ""]) + old_server.setData(0, Qt.ItemDataRole.UserRole, {"port": 7777}) + tree.addTopLevelItem(old_server) + + old_child = QTreeWidgetItem(["Worker", "ok", ""]) + old_child.setData( + 0, + Qt.ItemDataRole.UserRole, + {"type": "worker", "node_id": "worker_0"}, + ) + old_server.addChild(old_child) + old_server.setExpanded(True) + old_child.setSelected(True) + + coordinator = TreeRebuildCoordinator() + + def rebuild() -> None: + new_server = QTreeWidgetItem(["Port 7777", "ok", ""]) + new_server.setData(0, Qt.ItemDataRole.UserRole, {"port": 7777}) + tree.addTopLevelItem(new_server) + new_child = QTreeWidgetItem(["Worker", "ok", ""]) + new_child.setData( + 0, + Qt.ItemDataRole.UserRole, + {"type": "worker", "node_id": "worker_0"}, + ) + new_server.addChild(new_child) + + coordinator.rebuild(tree, rebuild) + + rebuilt_server = tree.topLevelItem(0) + rebuilt_child = rebuilt_server.child(0) + assert rebuilt_server.isExpanded() + assert rebuilt_child.isSelected() diff --git a/tests/unit/pyqt_gui/test_tree_state_adapter.py b/tests/unit/pyqt_gui/test_tree_state_adapter.py new file mode 100644 index 000000000..a755ffa57 --- /dev/null +++ b/tests/unit/pyqt_gui/test_tree_state_adapter.py @@ -0,0 +1,59 @@ +import pytest + +try: + from PyQt6.QtCore import Qt + from PyQt6.QtWidgets import QApplication, QTreeWidget, QTreeWidgetItem + + PYQT_AVAILABLE = True +except Exception: + PYQT_AVAILABLE = False + +from pyqt_reactive.widgets.shared import TreeStateAdapter + + +@pytest.mark.skipif(not PYQT_AVAILABLE, reason="PyQt6 not available") +def test_tree_state_adapter_restores_expansion_and_selection_by_key(): + app = QApplication.instance() + if app is None: + app = QApplication([]) + + tree = QTreeWidget() + tree.setHeaderLabels(["Server", "Status", "Info"]) + + server = QTreeWidgetItem(["Port 7777 - Execution Server", "✅", ""]) + server.setData(0, Qt.ItemDataRole.UserRole, {"port": 7777}) + tree.addTopLevelItem(server) + + worker = QTreeWidgetItem(["Worker worker_0", "⚙️", ""]) + worker.setData( + 0, + Qt.ItemDataRole.UserRole, + {"type": "worker", "node_id": "worker_0"}, + ) + server.addChild(worker) + + server.setExpanded(True) + worker.setSelected(True) + + adapter = TreeStateAdapter() + expansion_state = adapter.capture_expansion_state(tree) + selected_keys = adapter.capture_selected_keys(tree) + + tree.clear() + rebuilt_server = QTreeWidgetItem(["Port 7777 - Execution Server", "✅", ""]) + rebuilt_server.setData(0, Qt.ItemDataRole.UserRole, {"port": 7777}) + tree.addTopLevelItem(rebuilt_server) + + rebuilt_worker = QTreeWidgetItem(["Worker worker_0", "⚙️", ""]) + rebuilt_worker.setData( + 0, + Qt.ItemDataRole.UserRole, + {"type": "worker", "node_id": "worker_0"}, + ) + rebuilt_server.addChild(rebuilt_worker) + + adapter.restore_expansion_state(tree, expansion_state) + adapter.restore_selected_keys(tree, selected_keys) + + assert rebuilt_server.isExpanded() + assert rebuilt_worker.isSelected() diff --git a/tests/unit/pyqt_gui/test_zmq_server_info_parser.py b/tests/unit/pyqt_gui/test_zmq_server_info_parser.py new file mode 100644 index 000000000..7f37834c9 --- /dev/null +++ b/tests/unit/pyqt_gui/test_zmq_server_info_parser.py @@ -0,0 +1,87 @@ +from pyqt_reactive.services import ( + DefaultServerInfoParser, + ExecutionServerInfo, + ViewerServerInfo, + GenericServerInfo, + ServerKind, +) + + +def test_execution_server_payload_parses_to_typed_info(): + parser = DefaultServerInfoParser() + payload = { + "port": 7777, + "ready": True, + "server": "OpenHCSExecutionServer", + "log_file_path": "/tmp/server.log", + "workers": [ + { + "pid": 1234, + "status": "running", + "cpu_percent": 12.0, + "memory_mb": 256.0, + } + ], + "running_executions": [ + { + "execution_id": "exec-1", + "plate_id": "/tmp/p1", + "compile_only": True, + }, + ], + "queued_executions": [ + {"execution_id": "exec-2", "plate_id": "/tmp/p2", "queue_position": 1}, + ], + "compile_status": "compiled success", + "compile_message": "ok", + } + + info = parser.parse(payload) + + assert isinstance(info, ExecutionServerInfo) + assert info.kind == ServerKind.EXECUTION + assert info.port == 7777 + assert len(info.workers) == 1 + assert info.running_executions == ("exec-1",) + assert info.queued_executions == ("exec-2",) + assert info.running_execution_entries[0].plate_id == "/tmp/p1" + assert info.running_execution_entries[0].compile_only is True + assert info.queued_execution_entries[0].queue_position == 1 + assert info.compile_status is not None + assert info.compile_status.status_text == "compiled success" + assert info.compile_status.is_success + + +def test_viewer_server_payload_parses_to_typed_info(): + parser = DefaultServerInfoParser() + payload = { + "port": 7780, + "ready": True, + "server": "NapariViewerServer", + "log_file_path": "/tmp/napari.log", + "memory_mb": 1024.5, + "cpu_percent": 8.25, + } + + info = parser.parse(payload) + + assert isinstance(info, ViewerServerInfo) + assert info.kind == ServerKind.NAPARI + assert info.memory_mb == 1024.5 + assert info.cpu_percent == 8.25 + + +def test_unknown_server_payload_parses_to_generic_info(): + parser = DefaultServerInfoParser() + payload = { + "port": 9000, + "ready": False, + "server": "CustomServer", + "log_file_path": None, + } + + info = parser.parse(payload) + + assert isinstance(info, GenericServerInfo) + assert info.kind == ServerKind.GENERIC + assert info.server_name == "CustomServer" diff --git a/tests/unit/test_batch_submit_wait_engine.py b/tests/unit/test_batch_submit_wait_engine.py new file mode 100644 index 000000000..d5298c7dc --- /dev/null +++ b/tests/unit/test_batch_submit_wait_engine.py @@ -0,0 +1,88 @@ +import asyncio + +import pytest + +from zmqruntime.execution import ( + BatchSubmitWaitEngine, + CallbackBatchSubmitWaitPolicy, +) + + +def test_batch_submit_wait_engine_submits_then_waits(): + events: list[str] = [] + + async def submit(job: str) -> str: + events.append(f"submit:{job}") + return f"id:{job}" + + async def wait(submission_id: str, job: str) -> None: + events.append(f"wait:{job}:{submission_id}") + + policy = CallbackBatchSubmitWaitPolicy[str]( + submit_fn=submit, + wait_fn=wait, + job_key_fn=lambda job: job, + fail_fast_submit_value=True, + fail_fast_wait_value=True, + ) + + artifacts = asyncio.run(BatchSubmitWaitEngine[str]().run(["a", "b"], policy)) + + assert artifacts == {"a": "id:a", "b": "id:b"} + assert events == [ + "submit:a", + "submit:b", + "wait:a:id:a", + "wait:b:id:b", + ] + + +def test_batch_submit_wait_engine_non_fail_fast_wait_continues(): + errors: list[str] = [] + finals: list[str] = [] + + async def submit(job: str) -> str: + return f"id:{job}" + + async def wait(submission_id: str, job: str) -> None: + if job == "b": + raise RuntimeError("failed") + + policy = CallbackBatchSubmitWaitPolicy[str]( + submit_fn=submit, + wait_fn=wait, + job_key_fn=lambda job: job, + fail_fast_submit_value=True, + fail_fast_wait_value=False, + on_wait_error_fn=lambda job, error, _idx, _total: errors.append( + f"{job}:{error}" + ), + on_wait_finally_fn=lambda job, _idx, _total: finals.append(job), + ) + + artifacts = asyncio.run(BatchSubmitWaitEngine[str]().run(["a", "b"], policy)) + + assert artifacts == {"a": "id:a"} + assert errors == ["b:failed"] + assert finals == ["a", "b"] + + +def test_batch_submit_wait_engine_fail_fast_submit_raises(): + async def submit(job: str) -> str: + if job == "b": + raise RuntimeError("submit failed") + return f"id:{job}" + + async def wait(submission_id: str, job: str) -> None: + return None + + policy = CallbackBatchSubmitWaitPolicy[str]( + submit_fn=submit, + wait_fn=wait, + job_key_fn=lambda job: job, + fail_fast_submit_value=True, + fail_fast_wait_value=True, + ) + + with pytest.raises(RuntimeError, match="submit failed"): + asyncio.run(BatchSubmitWaitEngine[str]().run(["a", "b", "c"], policy)) diff --git a/tests/unit/test_execution_status_poller.py b/tests/unit/test_execution_status_poller.py new file mode 100644 index 000000000..23b147e38 --- /dev/null +++ b/tests/unit/test_execution_status_poller.py @@ -0,0 +1,73 @@ +from zmqruntime.execution import ( + ExecutionStatusPoller, + CallbackExecutionStatusPollPolicy, +) + + +def test_execution_status_poller_handles_running_then_complete(): + events: list[str] = [] + responses = iter( + [ + {"status": "ok", "execution": {"status": "queued"}}, + {"status": "ok", "execution": {"status": "running"}}, + {"status": "ok", "execution": {"status": "complete"}}, + ] + ) + + def poll_status(_execution_id: str): + return next(responses) + + policy = CallbackExecutionStatusPollPolicy( + poll_status_fn=poll_status, + poll_interval_seconds_value=0.0, + on_running_fn=lambda execution_id, _payload: events.append( + f"running:{execution_id}" + ), + on_terminal_fn=lambda execution_id, terminal_status, _payload: events.append( + f"terminal:{execution_id}:{terminal_status}" + ), + ) + + ExecutionStatusPoller().run("exec-1", policy) + + assert events == ["running:exec-1", "terminal:exec-1:complete"] + + +def test_execution_status_poller_handles_status_error(): + events: list[str] = [] + + policy = CallbackExecutionStatusPollPolicy( + poll_status_fn=lambda _execution_id: {"status": "error", "error": "unavailable"}, + poll_interval_seconds_value=0.0, + on_status_error_fn=lambda execution_id, message: events.append( + f"error:{execution_id}:{message}" + ), + ) + + ExecutionStatusPoller().run("exec-2", policy) + + assert events == ["error:exec-2:unavailable"] + + +def test_execution_status_poller_poll_exception_can_continue(): + events: list[str] = [] + state = {"count": 0} + + def poll_status(_execution_id: str): + state["count"] += 1 + if state["count"] == 1: + raise RuntimeError("transient") + return {"status": "ok", "execution": {"status": "cancelled"}} + + policy = CallbackExecutionStatusPollPolicy( + poll_status_fn=poll_status, + poll_interval_seconds_value=0.0, + on_poll_exception_fn=lambda _execution_id, _error: True, + on_terminal_fn=lambda execution_id, terminal_status, _payload: events.append( + f"terminal:{execution_id}:{terminal_status}" + ), + ) + + ExecutionStatusPoller().run("exec-3", policy) + + assert events == ["terminal:exec-3:cancelled"] diff --git a/tests/unit/test_materialization_core.py b/tests/unit/test_materialization_core.py index d6f3d8efa..c6f6f14c1 100644 --- a/tests/unit/test_materialization_core.py +++ b/tests/unit/test_materialization_core.py @@ -1,59 +1,26 @@ import json -import numpy as np import pytest from polystore.filemanager import FileManager from polystore.memory import MemoryStorageBackend -from openhcs.processing.materialization import materialize, regionprops_materializer -from openhcs.processing.materialization.core import _generate_output_path +from openhcs.processing.materialization import JsonOptions, MaterializationSpec, materialize @pytest.mark.unit -def test_generate_output_path_strips_roi_zip_compound_suffix() -> None: - base = "/tmp/A01_s001_w1_z001_t001_segmentation_masks_step7.roi.zip" - assert _generate_output_path(base, ".csv", ".csv") == "/tmp/A01_s001_w1_z001_t001_segmentation_masks_step7.csv" - - -@pytest.mark.unit -def test_regionprops_materializer_writes_roi_csv_json_with_intensity() -> None: +def test_materialize_strips_roi_zip_compound_suffix_for_json() -> None: fm = FileManager({"memory": MemoryStorageBackend()}) - labels = np.zeros((10, 10), dtype=np.int32) - labels[1:5, 1:5] = 1 # area 16 (kept) - labels[6:9, 6:9] = 2 # area 9 (filtered out by min_area=10) - - intensity = np.arange(100, dtype=np.float32).reshape(10, 10) - - spec = regionprops_materializer(intensity_source=None) - + spec = MaterializationSpec(JsonOptions(filename_suffix=".json")) out = materialize( spec, - data=[labels], - path="/tmp/test_regionprops.roi.zip", + data={"ok": True}, + path="/tmp/A01_test_output.roi.zip", filemanager=fm, backends=["memory"], backend_kwargs={}, - extra_inputs={"intensity": [intensity]}, ) - assert out == "/tmp/test_regionprops.json" - - # JSON summary - summary = json.loads(fm.load("/tmp/test_regionprops.json", "memory")) - assert summary["total_regions"] == 1 - assert summary["total_slices"] == 1 - - # CSV details should include intensity metrics - csv_content = fm.load("/tmp/test_regionprops_details.csv", "memory") - assert "mean_intensity" in csv_content - - # ROI archive stored as list of ROI objects (memory backend stores raw objects) - rois = fm.load("/tmp/test_regionprops_rois.roi.zip", "memory") - assert isinstance(rois, list) - assert len(rois) == 1 - assert rois[0].metadata["label"] == 1 - assert rois[0].metadata["slice_index"] == 0 - assert "mean_intensity" in rois[0].metadata - + assert out == "/tmp/A01_test_output.json" + assert fm.load(out, "memory") == json.dumps({"ok": True}, indent=2, default=str)