diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..ed251c2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,61 @@ +name: 🐞 Bug report +description: Create a report to help us improve +labels: [bug, "pending triage"] +body: + - type: markdown + attributes: + value: | + **Before You Start...** + + This form is only for submitting bug reports. Before opening one, please: + - Search existing issues β€” your problem may already be reported or fixed in the dev branch. + - For usage questions, join the [Discord Chat](https://discord.gg/T7uQJWnNgr). + - type: textarea + id: description + attributes: + label: Describe the bug + description: A clear and concise description of what the bug is. + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: Steps to reproduce the behavior. + placeholder: | + 1. Go to '...' + 2. Click on '...' + 3. See the error + validations: + required: true + - type: input + id: version + attributes: + label: Modly version + description: Which version of Modly are you running? (Help β†’ About, or package.json) + placeholder: 0.3.5 + validations: + required: true + - type: input + id: os + attributes: + label: Operating system + placeholder: Windows 11 / macOS 14 (arm64) / Ubuntu 24.04 + validations: + required: true + - type: textarea + id: logs + attributes: + label: Screenshots & logs + description: Add screenshots and any relevant log output to help explain the problem. + validations: + required: false + - type: checkboxes + id: checklist + attributes: + label: Confirmation + options: + - label: I have searched existing issues and this has not been reported yet. + required: true + - label: I can reproduce this on the latest version of Modly. + required: true diff --git a/.github/ISSUE_TEMPLATE/bug_report_extension.yml b/.github/ISSUE_TEMPLATE/bug_report_extension.yml new file mode 100644 index 0000000..584e43b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report_extension.yml @@ -0,0 +1,80 @@ +name: 🧩 Bug report β€” Official extension +description: Report a bug in one of Modly's official extensions +labels: [bug, extension, "pending triage"] +body: + - type: markdown + attributes: + value: | + **Before You Start...** + + This form is only for bugs in **official extensions** shipped with Modly. + For core app bugs, use the regular *Bug report* instead. + - Search existing issues β€” your problem may already be reported or fixed in a newer version. + - Make sure the extension is up to date β€” **extensions do not update automatically**, so a fix may already be available in a newer version. + - Try to **repair** the extension, or **uninstall and reinstall** it β€” this fixes most issues. + - For usage questions, join the [Discord Chat](https://discord.gg/T7uQJWnNgr). + - type: input + id: extension + attributes: + label: Which official extension? + description: Name of the official extension affected. + placeholder: e.g. modly-hunyuan3d-mini-extension + validations: + required: true + - type: textarea + id: description + attributes: + label: Describe the bug + description: A clear and concise description of what the bug is. + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: Steps to reproduce the behavior. + placeholder: | + 1. Open the extension '...' + 2. Apply '...' + 3. See the error + validations: + required: true + - type: input + id: version + attributes: + label: Modly version + description: Which version of Modly are you running? (Help β†’ About, or package.json) + placeholder: 0.3.5 + validations: + required: true + - type: input + id: os + attributes: + label: Operating system + placeholder: Windows 11 / macOS 14 (arm64) / Ubuntu 24.04 + validations: + required: true + - type: input + id: gpu + attributes: + label: GPU + description: Which GPU are you using? (Modly runs locally on your GPU) + placeholder: e.g. NVIDIA RTX 4070 / Apple M3 / AMD RX 7800 XT + validations: + required: true + - type: textarea + id: logs + attributes: + label: Screenshots & logs + description: Add screenshots and any relevant log output to help explain the problem. + validations: + required: false + - type: checkboxes + id: checklist + attributes: + label: Confirmation + options: + - label: I have searched existing issues and this has not been reported yet. + required: true + - label: I can reproduce this on the latest version of Modly. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..040468d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: πŸ’¬ Discord Chat + url: https://discord.gg/T7uQJWnNgr + about: Ask questions and chat with the Modly community in real time. + - name: ❀️ Sponsor Modly + url: https://github.com/sponsors/lightningpixel + about: Love Modly? Please consider supporting development via GitHub Sponsors. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..f622f75 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,38 @@ +name: ✨ Feature request +description: Suggest a new feature or improvement for Modly +labels: [enhancement, "pending triage"] +body: + - type: markdown + attributes: + value: | + **Before You Start...** + + Please search existing issues first β€” your idea may already be proposed. + - type: textarea + id: problem + attributes: + label: What problem does this solve? + description: Is your feature request related to a problem? Describe it. + validations: + required: true + - type: textarea + id: solution + attributes: + label: Proposed solution + description: Describe the solution or feature you'd like. + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Any alternative solutions or workarounds you've considered. + validations: + required: false + - type: checkboxes + id: checklist + attributes: + label: Confirmation + options: + - label: I have searched existing issues for this feature. + required: true diff --git a/.gitignore b/.gitignore index d7178cd..b3e1fa5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ api/*.pyc api/**/*.pyc api/.venv/ api/venv/ +tools/modly-cli/__pycache__/ +.hermes-e2e/ # Models (heavy, downloaded at runtime) /models/ @@ -33,3 +35,9 @@ Thumbs.db .env .env.local +# TypeScript incremental build output +*.tsbuildinfo + +# Local dev helpers +install.bat +launch.bat diff --git a/README.md b/README.md index b05a1ca..0e0ef79 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ **Local, open source, AI-powered image-to-3D mesh generation.** Turn any photo into a 3D model using open source AI models running entirely on your GPU. -Modly is a desktop application for Windows and Linux (macOS coming soon) +Modly is a desktop application for Windows, Linux, and Apple Silicon macOS. > Created by [Lightning Pixel](https://github.com/lightningpixel) @@ -19,15 +19,15 @@ Modly is a desktop application for Windows and Linux (macOS coming soon) ## Download -Head to the [Releases](../../releases/latest) page to download the latest installer for Windows or Linux. +Head to the [Releases](../../releases/latest) page to download the latest installer for Windows, Linux, or Apple Silicon macOS. Alternatively, you can clone the repository and run the app directly without installing: ```bash # Windows -launcher.bat +launch.bat -# Linux +# Linux / macOS ./launcher.sh ``` @@ -59,11 +59,28 @@ pip install -r requirements.txt npm run dev ``` +### 4. Test + +```bash +npm test +./node_modules/.bin/tsc --noEmit -p tsconfig.node.json +npm run build +``` + +## Platform notes + +- macOS support targets Apple Silicon only. +- macOS uses native window controls. Windows and Linux keep the existing custom controls. +- The top bar includes a live RAM indicator sourced from the main process. +- Workflow wiring is validated before run; invalid graphs stay in place and surface inline/toast warnings instead of dropping the current mesh view. +- Package Apple Silicon macOS with `npm run package:mac`. +- Imported meshes can be smoothed and decimated in-app; optimized results are written back into the workspace. + --- ## Extension system -Modly supports external AI model extensions. Each extension is a GitHub repository containing a `manifest.json` and a `generator.py`. +Modly supports external model and process extensions. Each extension is a GitHub repository containing a `manifest.json` plus the runtime entry files required by its type. ### Official extensions @@ -85,20 +102,36 @@ Modly supports external AI model extensions. Each extension is a GitHub reposito ![Enter extension URL](docs/install-extension.png) -**3.** Once the extension is installed, download the model or one of its variants. +**3.** If the extension exposes model nodes, download the model or one of its variants. Process extensions are ready once installation and setup complete. ![Install models](docs/install-models.png) --- -### Community +## Modly CLI -Join the [Discord server](https://discord.gg/BvjDCvS3yr) to stay up to date with the latest news, report bugs, and share feedback. +Agents and scripts can call a running Modly desktop app without using the UI via the stdlib-only CLI. The CLI is a thin helper over Modly's canonical automation concepts and keeps final machine-readable JSON on stdout: -Follow Modly and its development on X: +```bash +python tools/modly-cli/agent.py health +python tools/modly-cli/agent.py model list +python tools/modly-cli/agent.py workflow-run status +python tools/modly-cli/agent.py generate --image ./input.png --output ./export.glb +``` -- [Modly on X](https://x.com/modly3d) -- [Lightning Pixel on X](https://x.com/lightningpiixel) +Canonical commands are `health`, `model`, `workflow-run`, `capability`, and `process-run`. The friendly `generate` command starts `POST /workflow-runs/from-image`, polls the returned run, exports the final mesh when requested, and includes recovery metadata such as `workflow-run status ...` and `workflow-run cancel ...` in the JSON response. + +Compatibility and helper surfaces are intentionally separated: `legacy` wraps old `/generate/*` job endpoints, `dev serve-api` / `dev ensure-server` start only the FastAPI backend and do not prove Electron/Desktop bridge readiness, and `experimental comfy-image` / `experimental generate-from-workflow` are external ComfyUI orchestration helpers rather than the canonical Modly agent contract. Hidden helper aliases such as `status`, `export`, and `batch` remain parseable for scripts, but they are not presented as canonical root commands. + +`experimental generate-from-workflow --workflow --output ` treats `--output` as the final artifact location. When the ComfyUI workflow produces a downloadable 3D asset, the CLI downloads it directly; image-only workflows remain a compatibility path through Modly image-to-3D generation. + +See `tools/modly-cli/SKILL.md` for the agent workflow and output contract. + +--- + +### Community + +Join the [Discord server](https://discord.gg/BvjDCvS3yr) to stay up to date with the latest news, report bugs, and share feedback. --- diff --git a/api/main.py b/api/main.py index 1404bec..e73b3f2 100644 --- a/api/main.py +++ b/api/main.py @@ -9,7 +9,7 @@ from fastapi.responses import FileResponse from fastapi import HTTPException -from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs +from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs, agent @asynccontextmanager @@ -31,7 +31,7 @@ def filter(self, record): app = FastAPI( title="Modly API", - version="0.3.6", + version="0.4.0", lifespan=lifespan, ) @@ -40,6 +40,9 @@ def filter(self, record): allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], + # drei's SplatLoader reads Content-Length to size its buffers; cross-origin + # JS can only see it when the server explicitly exposes the header. + expose_headers=["Content-Length"], ) app.include_router(status.router) @@ -50,6 +53,7 @@ def filter(self, record): app.include_router(extensions.router, prefix="/extensions") app.include_router(export.router, prefix="/export") app.include_router(workflow_runs.router, prefix="/workflow-runs") +app.include_router(agent.router) # Serve generated files from workspace β€” dynamic so path changes take effect immediately @app.get("/workspace/{full_path:path}") diff --git a/api/mcp_server.py b/api/mcp_server.py new file mode 100644 index 0000000..fa9127c --- /dev/null +++ b/api/mcp_server.py @@ -0,0 +1,265 @@ +""" +Modly MCP Server +Exposes Modly's capabilities as MCP tools for external agents (Claude Desktop, Codex CLI, etc.). + +Usage: + python mcp_server.py + +Configuration in Claude Desktop (~/.config/claude/claude_desktop_config.json): + { + "mcpServers": { + "modly": { + "command": "python", + "args": ["C:/path/to/modly/desktop/api/mcp_server.py"] + } + } + } + +Requires Modly's FastAPI backend to be running on http://localhost:8765. +""" + +import asyncio +import mimetypes +import httpx +from mcp.server import Server +from mcp.server.stdio import stdio_server +from mcp.types import Tool, TextContent + +API_BASE = "http://localhost:8765" + +server = Server("modly") + + +@server.list_tools() +async def list_tools() -> list[Tool]: + return [ + Tool( + name="modly_list_models", + description="List all 3D generation models available in Modly (downloaded and ready to use).", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="modly_switch_model", + description="Switch the active 3D generation model in Modly.", + inputSchema={ + "type": "object", + "properties": { + "model_id": {"type": "string", "description": "The model ID to activate."}, + }, + "required": ["model_id"], + }, + ), + Tool( + name="modly_generate_from_image", + description="Generate a 3D mesh from a 2D image file. Returns a job_id to track progress.", + inputSchema={ + "type": "object", + "properties": { + "image_path": { + "type": "string", + "description": "Absolute path to the image file on disk.", + }, + "model_id": { + "type": "string", + "description": "Which model to use. If omitted, uses the currently active model.", + }, + "remesh": { + "type": "string", + "enum": ["quad", "triangle", "none"], + "description": "Remesh strategy after generation. Default: quad.", + }, + }, + "required": ["image_path"], + }, + ), + Tool( + name="modly_get_generation_status", + description="Poll the status of a 3D generation job. Call repeatedly until status is 'done' or 'error'.", + inputSchema={ + "type": "object", + "properties": { + "job_id": {"type": "string", "description": "Job ID returned by modly_generate_from_image."}, + }, + "required": ["job_id"], + }, + ), + Tool( + name="modly_decimate_mesh", + description="Reduce the polygon count of a mesh using quadric edge collapse decimation.", + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + "target_faces": { + "type": "integer", + "description": "Target number of faces after decimation (minimum 100).", + }, + }, + "required": ["path", "target_faces"], + }, + ), + Tool( + name="modly_smooth_mesh", + description="Apply Laplacian smoothing to a mesh. More iterations = smoother surface but less detail.", + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + "iterations": { + "type": "integer", + "description": "Number of smoothing iterations (1–20).", + }, + }, + "required": ["path", "iterations"], + }, + ), + Tool( + name="modly_import_mesh", + description="Import a mesh file from disk into Modly's workspace (.glb, .obj, .stl, .ply).", + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Absolute path to the mesh file on disk.", + }, + }, + "required": ["path"], + }, + ), + Tool( + name="modly_unload_models", + description="Unload all 3D generation models from GPU VRAM. Useful before running VRAM-intensive tasks.", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="modly_get_settings", + description="Get the current Modly settings (models directory, workspace directory).", + inputSchema={"type": "object", "properties": {}}, + ), + ] + + +@server.call_tool() +async def call_tool(name: str, arguments: dict) -> list[TextContent]: + async with httpx.AsyncClient(timeout=60.0) as client: + try: + result = await _dispatch(client, name, arguments) + except httpx.ConnectError: + result = ( + "Cannot connect to Modly API at http://localhost:8765. " + "Make sure Modly is running." + ) + except httpx.HTTPStatusError as e: + result = f"Modly API error {e.response.status_code}: {e.response.text[:300]}" + except Exception as e: + result = f"Error: {e}" + + return [TextContent(type="text", text=result)] + + +async def _dispatch(client: httpx.AsyncClient, name: str, args: dict) -> str: + if name == "modly_list_models": + r = await client.get(f"{API_BASE}/model/all") + r.raise_for_status() + models = [m for m in r.json() if m.get("downloaded")] + if not models: + return "No models downloaded yet. Download one from the Models tab in Modly." + return "\n".join(f"- {m['id']}: {m.get('name', m['id'])}" for m in models) + + elif name == "modly_switch_model": + r = await client.post(f"{API_BASE}/model/switch", params={"model_id": args["model_id"]}) + r.raise_for_status() + return f"Switched active model to: {args['model_id']}" + + elif name == "modly_generate_from_image": + image_path: str = args["image_path"] + with open(image_path, "rb") as f: + img_bytes = f.read() + mime = mimetypes.guess_type(image_path)[0] or "image/png" + filename = image_path.replace("\\", "/").split("/")[-1] + + form_data = { + "remesh": args.get("remesh", "quad"), + } + if args.get("model_id"): + form_data["model_id"] = args["model_id"] + + r = await client.post( + f"{API_BASE}/generate/from-image", + files={"image": (filename, img_bytes, mime)}, + data=form_data, + timeout=30.0, + ) + r.raise_for_status() + job_id = r.json()["job_id"] + return ( + f"Generation started. Job ID: {job_id}\n" + f"Use modly_get_generation_status with this ID to track progress." + ) + + elif name == "modly_get_generation_status": + r = await client.get(f"{API_BASE}/generate/status/{args['job_id']}") + r.raise_for_status() + s = r.json() + parts = [f"Status: {s['status']}", f"Progress: {s.get('progress', 0)}%"] + if s.get("step"): + parts.append(f"Step: {s['step']}") + if s.get("output_url"): + parts.append(f"Output: {s['output_url']}") + if s.get("error"): + parts.append(f"Error: {s['error']}") + return " | ".join(parts) + + elif name == "modly_decimate_mesh": + r = await client.post( + f"{API_BASE}/optimize/mesh", + json={"path": args["path"], "target_faces": args["target_faces"]}, + ) + r.raise_for_status() + data = r.json() + return f"Decimated mesh to {data.get('face_count', '?')} faces. New file: {data.get('url', '')}" + + elif name == "modly_smooth_mesh": + r = await client.post( + f"{API_BASE}/optimize/smooth", + json={"path": args["path"], "iterations": args["iterations"]}, + ) + r.raise_for_status() + data = r.json() + return f"Smoothed mesh ({args['iterations']} iterations). New file: {data.get('url', '')}" + + elif name == "modly_import_mesh": + r = await client.post(f"{API_BASE}/optimize/import-by-path", json={"path": args["path"]}) + r.raise_for_status() + data = r.json() + return f"Mesh imported. URL: {data.get('url', '')}" + + elif name == "modly_unload_models": + r = await client.post(f"{API_BASE}/model/unload-all") + r.raise_for_status() + return "All 3D generation models unloaded from VRAM." + + elif name == "modly_get_settings": + r = await client.get(f"{API_BASE}/settings/paths") + r.raise_for_status() + data = r.json() + return f"Models directory: {data.get('models_dir')}\nWorkspace directory: {data.get('workspace_dir')}" + + else: + return f"Unknown tool: {name}" + + +async def main(): + async with stdio_server() as (read_stream, write_stream): + await server.run(read_stream, write_stream, server.create_initialization_options()) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/api/requirements.txt b/api/requirements.txt index 556a70c..975f19c 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,8 +1,12 @@ # Web server -fastapi==0.115.6 +fastapi>=0.115.6 uvicorn[standard]==0.34.0 python-multipart==0.0.20 +# Agent & MCP +httpx>=0.27.0 +mcp>=1.0.0 + # Mesh processing (optimize + export endpoints) trimesh>=4.5.0 pymeshlab>=2023.12 diff --git a/api/routers/agent.py b/api/routers/agent.py new file mode 100644 index 0000000..3eeb2a7 --- /dev/null +++ b/api/routers/agent.py @@ -0,0 +1,517 @@ +""" +Agent chat endpoint β€” runs an Ollama-powered tool-use loop against Modly's API. +""" +import re +import uuid +import httpx +from fastapi import APIRouter +from pydantic import BaseModel + +router = APIRouter(prefix="/agent", tags=["agent"]) + +MODLY_API = "http://localhost:8765" + +SYSTEM_PROMPT = """\ +You are Modly's built-in AI assistant, specialized in 3D modeling and workflow automation. +You help users generate 3D models from images, optimize meshes, and manage workflows directly inside the Modly application. + +## Available tools + +- **list_models** β€” List all downloaded 3D generation models ready to use. +- **unload_models** β€” Unload all 3D generation models from GPU VRAM to free memory. +- **get_mesh_info** β€” Get info about the current mesh in the 3D viewer (path, triangle count). +- **decimate_mesh(path, target_faces)** β€” Reduce the polygon count of a mesh. +- **smooth_mesh(path, iterations)** β€” Apply Laplacian smoothing to a mesh. +- **get_generation_status(job_id)** β€” Poll the status of an ongoing 3D generation job. +- **list_workflows** β€” List all available workflows in Modly. +- **run_workflow(workflow_id)** β€” Execute a workflow in Modly by its ID. If the user attached an image in their message, it will automatically be used as the workflow's input image. +- **create_workflow(name, input_type, steps, description?)** β€” Create a new workflow from an ordered list of processing steps. Each step references an extension by its exact `id` and may override its params. The steps run in sequence, the output of one feeding the next. The input source is one of exactly three nodes β€” `image` (Image), `text` (Text), or `mesh` (Load 3D Mesh) β€” and an Add-to-Scene output node is appended automatically. + +## Rules + +- Always use tools to act on the scene β€” never just describe what you would do. +- If you need the current mesh path, call get_mesh_info first. +- If you need to run a workflow but don't know the ID, call list_workflows first. +- To create a workflow, ONLY use extension ids listed under "Available extensions" in the context. Never invent an id. Chain steps so each step's input type matches the previous step's output type. +- For a workflow's input, `input_type` MUST be exactly one of: `image`, `text`, or `mesh`. These map to the Image, Text, and Load 3D Mesh nodes. Never invent another input. Pick the one matching the first step's expected input. +- After each tool call, give a short one-sentence summary of what was done. +- Always reply in the same language the user is writing in. +- Be concise. No unnecessary explanations.\ +""" + +TOOLS = [ + { + "type": "function", + "function": { + "name": "list_models", + "description": "List all available 3D generation models that are downloaded and ready.", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "unload_models", + "description": "Unload all 3D generation models from VRAM to free GPU memory.", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "get_mesh_info", + "description": "Get information about the current mesh loaded in the 3D viewer (triangle count, path, etc.).", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "decimate_mesh", + "description": "Reduce the polygon count of the current mesh using quadric edge collapse.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file (e.g. 'Default/mesh.glb'). Use get_mesh_info to obtain it.", + }, + "target_faces": { + "type": "integer", + "description": "Target number of faces after decimation.", + }, + }, + "required": ["path", "target_faces"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "smooth_mesh", + "description": "Apply Laplacian smoothing to the current mesh.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + "iterations": { + "type": "integer", + "description": "Number of smoothing iterations (1–20). More = smoother but loses detail.", + }, + }, + "required": ["path", "iterations"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_generation_status", + "description": "Poll the status of an ongoing 3D generation job.", + "parameters": { + "type": "object", + "properties": { + "job_id": {"type": "string", "description": "Job ID returned by a previous generation call."}, + }, + "required": ["job_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "list_workflows", + "description": "List all workflows available in Modly.", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "run_workflow", + "description": "Execute a Modly workflow by its ID. The workflow runs in the background; progress is shown in the app.", + "parameters": { + "type": "object", + "properties": { + "workflow_id": {"type": "string", "description": "The workflow ID to execute. Use list_workflows to get available IDs."}, + }, + "required": ["workflow_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "create_workflow", + "description": ( + "Create a new Modly workflow from an ordered list of steps. " + "Each step references an extension by its exact id (see 'Available extensions' in context). " + "Steps run in sequence; do not include the input itself as a step." + ), + "parameters": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "Short human-readable name for the workflow."}, + "description": {"type": "string", "description": "Optional one-line description of what the workflow does."}, + "input_type": { + "type": "string", + "enum": ["image", "text", "mesh"], + "description": ( + "The workflow's input source node. Exactly one of: " + "'image' (Image node), 'text' (Text node), " + "'mesh' (Load 3D Mesh node, uses the current scene mesh). " + "Never use any other value." + ), + }, + "steps": { + "type": "array", + "description": "Ordered processing steps. Each runs after the previous one.", + "items": { + "type": "object", + "properties": { + "extension_id": { + "type": "string", + "description": "Exact extension id from 'Available extensions' (e.g. 'mesh-optimizer/optimize').", + }, + "params": { + "type": "object", + "description": "Optional param overrides, keyed by param id. Omit to use defaults.", + }, + }, + "required": ["extension_id"], + }, + }, + }, + "required": ["name", "input_type", "steps"], + }, + }, + }, +] + + +# Input kinds the agent may pick, mapped to the real Modly source-node types. +# Keep this in sync with the node palette in WorkflowsPage.tsx. +INPUT_NODES = { + "image": {"type": "imageNode", "data": {"enabled": True, "params": {}, "showInGenerate": True}}, + "text": {"type": "textNode", "data": {"enabled": True, "params": {}}}, + "mesh": {"type": "meshNode", "data": {"enabled": True, "params": {"source": "current"}}}, +} + + +def _build_workflow_graph(name: str, description: str, input_type: str, steps: list[dict]) -> dict: + """Assemble a Modly workflow graph (nodes + edges) from a simplified step spec. + + Layout: one source node (Image / Text / Load 3D Mesh), one extensionNode per + step, then an Add-to-Scene output node, all wired in a single linear chain with + workflowEdge edges. id/timestamps are left for the frontend to stamp + (crypto.randomUUID + ISO date), matching how the Workflows tab creates workflows. + """ + spec = INPUT_NODES.get(input_type, INPUT_NODES["image"]) + input_node = { + "id": uuid.uuid4().hex[:8], + "type": spec["type"], + "position": {"x": 250, "y": 50}, + "data": {**spec["data"]}, + } + + ext_nodes = [] + for i, step in enumerate(steps): + ext_nodes.append({ + "id": uuid.uuid4().hex[:8], + "type": "extensionNode", + "position": {"x": 250, "y": 150 + i * 200}, + "data": { + "extensionId": step["extension_id"], + "enabled": True, + "params": step.get("params") or {}, + }, + }) + + output_node = { + "id": uuid.uuid4().hex[:8], + "type": "outputNode", + "position": {"x": 250, "y": 150 + len(steps) * 200}, + "data": {"enabled": True, "params": {}}, + } + + all_nodes = [input_node, *ext_nodes, output_node] + edges = [ + { + "id": f"e-{all_nodes[i]['id']}-{all_nodes[i + 1]['id']}", + "source": all_nodes[i]["id"], + "target": all_nodes[i + 1]["id"], + "type": "workflowEdge", + } + for i in range(len(all_nodes) - 1) + ] + + return {"name": name, "description": description, "nodes": all_nodes, "edges": edges} + + +async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, dict | None]: + """Execute a tool and return (result_text, action_payload). + action_payload carries data the frontend needs to react (e.g. new mesh URL). + """ + async with httpx.AsyncClient(timeout=60.0) as client: + try: + if name == "list_models": + r = await client.get(f"{MODLY_API}/model/all") + r.raise_for_status() + models = [m for m in r.json() if m.get("downloaded")] + if not models: + return "No models downloaded yet.", None + lines = "\n".join(f"- {m['id']}: {m.get('name', m['id'])}" for m in models) + return f"Available models:\n{lines}", None + + elif name == "unload_models": + await client.post(f"{MODLY_API}/model/unload-all") + return "All 3D generation models have been unloaded from VRAM.", None + + elif name == "get_mesh_info": + mesh_path = context.get("currentMeshPath") + mesh_triangles = context.get("meshTriangles") + if not mesh_path: + return "No mesh currently loaded in the viewer.", None + info = f"Current mesh: {mesh_path}" + if mesh_triangles: + info += f" ({mesh_triangles:,} triangles)" + return info, None + + elif name == "decimate_mesh": + r = await client.post( + f"{MODLY_API}/optimize/mesh", + json={"path": arguments["path"], "target_faces": arguments["target_faces"]}, + ) + r.raise_for_status() + data = r.json() + payload = {"type": "mesh_update", "url": data["url"], "face_count": data.get("face_count")} + return f"Decimated to {data.get('face_count', '?')} faces.", payload + + elif name == "smooth_mesh": + r = await client.post( + f"{MODLY_API}/optimize/smooth", + json={"path": arguments["path"], "iterations": arguments["iterations"]}, + ) + r.raise_for_status() + data = r.json() + payload = {"type": "mesh_update", "url": data["url"]} + return f"Smoothed mesh ({arguments['iterations']} iterations).", payload + + elif name == "get_generation_status": + r = await client.get(f"{MODLY_API}/generate/status/{arguments['job_id']}") + r.raise_for_status() + s = r.json() + text = f"Status: {s['status']}, Progress: {s.get('progress', 0)}%" + if s.get("step"): + text += f", Step: {s['step']}" + if s.get("output_url"): + text += f", Output: {s['output_url']}" + return text, None + + elif name == "list_workflows": + workflows = context.get("workflows", []) + if not workflows: + return "No workflows found. Create one in the Workflows tab.", None + lines = "\n".join(f"- {w['id']}: {w['name']}" for w in workflows) + return f"Available workflows:\n{lines}", None + + elif name == "run_workflow": + workflow_id = arguments["workflow_id"] + workflows = context.get("workflows", []) + match = next((w for w in workflows if w["id"] == workflow_id), None) + if not match: + return f"Workflow '{workflow_id}' not found. Use list_workflows to see available workflows.", None + payload = {"type": "run_workflow", "workflow_id": workflow_id, "workflow_name": match["name"]} + return f"Executing workflow '{match['name']}'…", payload + + elif name == "create_workflow": + steps = arguments.get("steps") or [] + if not steps: + return "A workflow needs at least one step. Specify the extensions to chain.", None + + input_type = arguments.get("input_type") or "image" + if input_type not in INPUT_NODES: + return ( + f"Invalid input_type '{input_type}'. Use exactly one of: " + f"image (Image node), text (Text node), mesh (Load 3D Mesh node).", + None, + ) + + extensions = context.get("extensions", []) + valid_ids = {e["id"] for e in extensions} + if valid_ids: + unknown = [s.get("extension_id") for s in steps if s.get("extension_id") not in valid_ids] + if unknown: + avail = ", ".join(sorted(valid_ids)) or "(none installed)" + return ( + f"Unknown extension id(s): {', '.join(map(str, unknown))}. " + f"Use only these: {avail}.", + None, + ) + + wf = _build_workflow_graph( + name=arguments.get("name") or "New Workflow", + description=arguments.get("description") or "", + input_type=input_type, + steps=steps, + ) + payload = {"type": "create_workflow", "workflow": wf} + return f"Created workflow '{wf['name']}' with {len(steps)} step(s).", payload + + else: + return f"Unknown tool: {name}", None + + except httpx.HTTPStatusError as e: + return f"API error {e.response.status_code}: {e.response.text[:200]}", None + except Exception as e: + return f"Error: {e}", None + + +class ChatMessage(BaseModel): + role: str + content: str + images: list[str] = [] + + +class AgentChatRequest(BaseModel): + messages: list[ChatMessage] + ollama_url: str = "http://localhost:11434" + model: str = "qwen2.5:3b" + context: dict = {} + thinking: str = "auto" # "auto" | "on" | "off" + + +class ActionDone(BaseModel): + tool: str + result: str + payload: dict | None = None + + +class AgentChatResponse(BaseModel): + message: str + actions: list[ActionDone] = [] + thinking: str | None = None + + +def _extract_thinking(msg: dict) -> tuple[str, str | None]: + """Return (clean_content, thinking_text). Handles both Ollama native field and tags.""" + content = msg.get("content", "") + thinking = msg.get("thinking") or None + if not thinking: + match = re.search(r"(.*?)", content, re.DOTALL) + if match: + thinking = match.group(1).strip() + content = (content[: match.start()] + content[match.end() :]).strip() + return content, thinking + + +@router.get("/models") +async def list_ollama_models(ollama_url: str = "http://localhost:11434"): + async with httpx.AsyncClient(timeout=5.0) as client: + try: + r = await client.get(f"{ollama_url}/api/tags") + r.raise_for_status() + models = [m["name"] for m in r.json().get("models", [])] + return {"models": models} + except Exception: + return {"models": []} + + +@router.post("/chat", response_model=AgentChatResponse) +async def agent_chat(request: AgentChatRequest): + messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}] + + # Inject scene context so the LLM knows current state + if request.context: + ctx_lines = [] + if request.context.get("currentMeshPath"): + ctx_lines.append(f"Current mesh path: {request.context['currentMeshPath']}") + if request.context.get("meshTriangles"): + ctx_lines.append(f"Current mesh triangles: {request.context['meshTriangles']:,}") + if ctx_lines: + messages.append({ + "role": "system", + "content": "Scene context:\n" + "\n".join(ctx_lines), + }) + + extensions = request.context.get("extensions") or [] + if extensions: + ext_lines = [ + f"- {e['id']} ({e.get('input', '?')}β†’{e.get('output', '?')}): {e.get('name', e['id'])}" + for e in extensions + ] + messages.append({ + "role": "system", + "content": ( + "Available extensions (use the exact id when creating workflows):\n" + + "\n".join(ext_lines) + ), + }) + + for m in request.messages: + entry: dict = {"role": m.role, "content": m.content} + if m.images: + entry["images"] = m.images + messages.append(entry) + + actions_done: list[ActionDone] = [] + all_thinking: list[str] = [] + + # Build Ollama think param + ollama_extra: dict = {} + if request.thinking == "on": + ollama_extra["think"] = True + elif request.thinking == "off": + ollama_extra["think"] = False + + async with httpx.AsyncClient(timeout=120.0) as client: + for _ in range(10): # max tool-call rounds + r = await client.post( + f"{request.ollama_url}/api/chat", + json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": False, **ollama_extra}, + ) + + if r.status_code != 200: + return AgentChatResponse( + message=f"Ollama error ({r.status_code}). Is Ollama running at {request.ollama_url}?", + ) + + msg = r.json()["message"] + messages.append(msg) + + clean_content, thinking_text = _extract_thinking(msg) + if thinking_text: + all_thinking.append(thinking_text) + + tool_calls = msg.get("tool_calls") or [] + if not tool_calls: + combined_thinking = "\n\n---\n\n".join(all_thinking) if all_thinking else None + return AgentChatResponse( + message=clean_content, + actions=actions_done, + thinking=combined_thinking, + ) + + for tc in tool_calls: + fn = tc["function"] + result_text, payload = await execute_tool(fn["name"], fn.get("arguments") or {}, request.context) + actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload)) + messages.append({"role": "tool", "content": result_text}) + + has_workflow = any(a.tool == "run_workflow" for a in actions_done) + if has_workflow: + # Unload LLM from VRAM immediately so the workflow has full GPU memory + try: + await client.post( + f"{request.ollama_url}/api/generate", + json={"model": request.model, "keep_alive": 0}, + timeout=5.0, + ) + except Exception: + pass + + combined_thinking = "\n\n---\n\n".join(all_thinking) if all_thinking else None + return AgentChatResponse(message="Reached maximum tool iterations.", actions=actions_done, thinking=combined_thinking) diff --git a/api/routers/generation.py b/api/routers/generation.py index bdd0492..ad59899 100644 --- a/api/routers/generation.py +++ b/api/routers/generation.py @@ -127,7 +127,8 @@ async def _run_generation(job_id: str, image_bytes: bytes, params: dict, collect job.status = "running" def progress_cb(pct: int, step: str = "") -> None: - job.progress = pct + if pct > job.progress: + job.progress = pct if step: job.step = step @@ -193,7 +194,11 @@ def progress_cb(pct: int, step: str = "") -> None: if job_id in _cancelled: return tb = traceback.format_exc() - print(f"[Generation ERROR] {exc}\n{tb}") + msg = f"[Generation ERROR] {exc}\n{tb}" + try: + print(msg) + except UnicodeEncodeError: + print(msg.encode("ascii", errors="replace").decode("ascii")) job.status = "error" job.error = tb.strip() _completed_at[job_id] = time.monotonic() diff --git a/api/routers/model.py b/api/routers/model.py index 6d24871..4f04718 100644 --- a/api/routers/model.py +++ b/api/routers/model.py @@ -1,6 +1,13 @@ import asyncio import json +import time +import os +import socket +import threading +from pathlib import Path from typing import Optional +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse from services.generator_registry import generator_registry, MODELS_DIR @@ -8,6 +15,40 @@ router = APIRouter(tags=["model"]) +class DownloadPaused(Exception): + pass + + +class DownloadCancelled(Exception): + pass + + +_download_controls: dict[str, dict[str, threading.Event]] = {} + + +def _download_control(model_id: str) -> dict[str, threading.Event]: + """Return the current control for pause/cancel endpoints.""" + control = _download_controls.get(model_id) + if control is None: + control = {"pause": threading.Event(), "cancel": threading.Event()} + _download_controls[model_id] = control + return control + + +def _new_download_control(model_id: str) -> dict[str, threading.Event]: + """Create a fresh control for a new download session.""" + control: dict[str, threading.Event] = {"pause": threading.Event(), "cancel": threading.Event()} + _download_controls[model_id] = control + return control + + +def _check_download_control(control: dict[str, threading.Event]) -> None: + if control["cancel"].is_set(): + raise DownloadCancelled() + if control["pause"].is_set(): + raise DownloadPaused() + + @router.get("/status") async def model_status(): """Status of the active model.""" @@ -67,16 +108,37 @@ async def unload_model(model_id: str): return {"unloaded": True} # already not loaded, that's fine +@router.post("/hf-download/pause") +async def pause_hf_download(model_id: str): + control = _download_control(model_id) + control["pause"].set() + return {"paused": True} + + +@router.post("/hf-download/cancel") +async def cancel_hf_download(model_id: str): + control = _download_control(model_id) + control["cancel"].set() + return {"cancelled": True} + + @router.get("/hf-download") -async def hf_download(repo_id: str, model_id: str, skip_prefixes: Optional[str] = None, token: Optional[str] = None): +async def hf_download( + repo_id: str, + model_id: str, + skip_prefixes: Optional[str] = None, + include_prefixes: Optional[str] = None, + token: Optional[str] = None, +): """ Streams a HuggingFace Hub model download via SSE. Downloads into MODELS_DIR / model_id applying the filtering - declared in the extension manifest (hf_skip_prefixes). + declared in the extension manifest (hf_skip_prefixes / hf_include_prefixes). - skip_prefixes: JSON-encoded list of path prefixes to exclude (passed from Electron). - token: HuggingFace access token for gated repos (passed from Electron settings). - Falls back to registry manifest if not provided. + skip_prefixes: JSON-encoded list of path prefixes to exclude. + include_prefixes: JSON-encoded list of path prefixes to include (whitelist). + token: HuggingFace access token for gated repos (from Electron settings). + All three fall back to the extension's manifest / environment when not supplied. SSE format: data: {"percent": 0-100, "file": "...", "status": "..."} """ @@ -95,8 +157,20 @@ async def hf_download(repo_id: str, model_id: str, skip_prefixes: Optional[str] except KeyError: skip_list = [] - # Token: explicit param > env var + if include_prefixes: + try: + include_list = _json.loads(include_prefixes) + except Exception: + include_list = [] + else: + try: + include_list = generator_registry.get_manifest(model_id).get("hf_include_prefixes", []) + except KeyError: + include_list = [] + + # Token: explicit query param > env var > None hf_token = token or os.environ.get("HUGGING_FACE_HUB_TOKEN") or os.environ.get("HF_TOKEN") or None + control = _new_download_control(model_id) async def stream(): loop = asyncio.get_running_loop() @@ -106,11 +180,13 @@ def _fmt(data: dict) -> str: try: yield _fmt({"percent": 0, "status": "Listing repository files..."}) + _check_download_control(control) def _list_files(): from huggingface_hub import list_repo_files return [ f for f in list_repo_files(repo_id, token=hf_token) + if (not include_list or any(f.startswith(p) for p in include_list)) if not any(f.startswith(p) for p in skip_list) ] @@ -123,29 +199,240 @@ def _list_files(): yield _fmt({"percent": 1, "status": f"Downloading {total} files..."}) - from huggingface_hub import hf_hub_download + from huggingface_hub import hf_hub_url for i, filename in enumerate(files): - def _dl(f=filename): - hf_hub_download( - repo_id=repo_id, - filename=f, - local_dir=dest_dir, - local_dir_use_symlinks=False, + _check_download_control(control) + yield _fmt({ + "percent": 1 + round(i / total * 94), + "file": filename, + "fileIndex": i + 1, + "totalFiles": total, + "status": f"Starting {filename}", + "bytesDownloaded": 0, + "stalledSeconds": 0, + }) + + base_pct = 1 + round(i / total * 94) + queue: asyncio.Queue[dict] = asyncio.Queue() + + def _progress(msg: dict) -> None: + loop.call_soon_threadsafe(queue.put_nowait, msg) + + url = hf_hub_url(repo_id=repo_id, filename=filename) + dl_future = loop.run_in_executor( + None, + lambda: _download_file_streamed( + url=url, + filename=filename, + dest_dir=dest_dir, + file_index=i + 1, + total_files=total, + base_percent=base_pct, + progress_cb=_progress, + control=control, token=hf_token, - ) + ), + ) + + while not dl_future.done(): + try: + msg = await asyncio.wait_for(queue.get(), timeout=2.0) + except asyncio.TimeoutError: + continue + else: + yield _fmt(msg) - await loop.run_in_executor(None, _dl) + final_size = await dl_future + _check_download_control(control) # Reserve 1-95 for file downloads, leave 95-100 for finalisation pct = 1 + round((i + 1) / total * 94) - yield _fmt({"percent": pct, "file": filename, "fileIndex": i + 1, "totalFiles": total}) + yield _fmt({ + "percent": pct, + "file": filename, + "fileIndex": i + 1, + "totalFiles": total, + "status": "Downloaded", + "bytesDownloaded": final_size, + "stalledSeconds": 0, + }) yield _fmt({"percent": 100, "status": "done"}) + except DownloadPaused: + yield _fmt({"paused": True, "status": "paused"}) + except DownloadCancelled: + # Remove only partial files; completed files are preserved so the + # next download can resume from where it left off. + for part in Path(dest_dir).rglob("*.part"): + part.unlink(missing_ok=True) + yield _fmt({"cancelled": True, "status": "cancelled"}) except Exception as exc: yield _fmt({"error": str(exc)}) + finally: + # Only remove the control if it still belongs to this session. + if _download_controls.get(model_id) is control: + _download_controls.pop(model_id, None) return StreamingResponse(stream(), media_type="text/event-stream") +def _download_file_streamed( + *, + url: str, + filename: str, + dest_dir: str, + file_index: int, + total_files: int, + base_percent: int, + progress_cb, + control: dict[str, threading.Event], + token: Optional[str] = None, +) -> int: + final_path = Path(dest_dir) / filename + temp_path = final_path.with_suffix(final_path.suffix + ".part") + final_path.parent.mkdir(parents=True, exist_ok=True) + + if final_path.exists(): + return final_path.stat().st_size + + # Explicit token (from caller) > env vars > none + hf_token = ( + token + or os.environ.get("HF_TOKEN") + or os.environ.get("HUGGINGFACE_HUB_TOKEN") + or os.environ.get("HUGGING_FACE_HUB_TOKEN") + ) + headers = {"User-Agent": "modly/0.3.1"} + if hf_token: + headers["Authorization"] = f"Bearer {hf_token}" + + retries = 3 + backoff = 2.0 + last_error: Exception | None = None + + for attempt in range(1, retries + 1): + try: + _check_download_control(control) + existing_bytes = temp_path.stat().st_size if temp_path.exists() else 0 + request_headers = dict(headers) + request_url = url + if existing_bytes > 0: + request_url = _resolve_direct_download_url(url, headers) + request_headers["Range"] = f"bytes={existing_bytes}-" + + request = Request(request_url, headers=request_headers) + with urlopen(request, timeout=30) as response: + resumed = existing_bytes > 0 and getattr(response, "status", None) == 206 + if existing_bytes > 0 and not resumed: + temp_path.unlink(missing_ok=True) + existing_bytes = 0 + + total_bytes = _response_total_bytes(response.headers, existing_bytes if resumed else 0) + bytes_downloaded = existing_bytes + last_emit = 0.0 + chunk_size = 1024 * 1024 + mode = "ab" if resumed else "wb" + + progress_cb({ + "percent": base_percent, + "file": filename, + "fileIndex": file_index, + "totalFiles": total_files, + "status": _download_status(bytes_downloaded, total_bytes, attempt, retries, resumed=resumed), + "bytesDownloaded": bytes_downloaded, + "totalBytes": total_bytes, + "stalledSeconds": 0, + }) + + with open(temp_path, mode) as out: + while True: + _check_download_control(control) + try: + chunk = response.read(chunk_size) + except socket.timeout as exc: + raise TimeoutError(f"Timed out while downloading {filename}") from exc + + if not chunk: + break + + out.write(chunk) + bytes_downloaded += len(chunk) + + now = time.monotonic() + if now - last_emit >= 0.5: + progress_cb({ + "percent": base_percent, + "file": filename, + "fileIndex": file_index, + "totalFiles": total_files, + "status": _download_status(bytes_downloaded, total_bytes, attempt, retries, resumed=resumed), + "bytesDownloaded": bytes_downloaded, + "totalBytes": total_bytes, + "stalledSeconds": 0, + }) + last_emit = now + + temp_path.replace(final_path) + return bytes_downloaded + + except (HTTPError, URLError, TimeoutError, OSError) as exc: + last_error = exc + preserved_bytes = temp_path.stat().st_size if temp_path.exists() else 0 + progress_cb({ + "percent": base_percent, + "file": filename, + "fileIndex": file_index, + "totalFiles": total_files, + "status": f"Retrying after error ({attempt}/{retries})…", + "bytesDownloaded": preserved_bytes, + "stalledSeconds": 0, + }) + if attempt >= retries: + break + time.sleep(backoff) + backoff *= 2 + + raise RuntimeError(f"Failed to download {filename}: {last_error}") + + +def _resolve_direct_download_url(url: str, headers: dict[str, str]) -> str: + # HEAD request: follows redirects to get the final CDN URL without downloading the body. + request = Request(url, headers=headers, method="HEAD") + with urlopen(request, timeout=30) as response: + return response.geturl() + + +def _parse_content_length(raw: Optional[str]) -> Optional[int]: + if not raw: + return None + try: + return int(raw) + except (TypeError, ValueError): + return None + + +def _download_status(downloaded: int, total: Optional[int], attempt: int, retries: int, resumed: bool = False) -> str: + prefix = "Resuming…" if resumed and downloaded > 0 else "Downloading…" + if total and total > 0: + pct = min(100, round(downloaded / total * 100)) + return f"{prefix} {pct}%" + if retries > 1 and attempt > 1: + return f"{prefix} retry {attempt}/{retries}" + return prefix + + +def _response_total_bytes(headers, already_downloaded: int) -> Optional[int]: + content_range = headers.get("Content-Range") + if content_range and "/" in content_range: + total_raw = content_range.split("/")[-1].strip() + try: + return int(total_raw) + except (TypeError, ValueError): + pass + + content_length = _parse_content_length(headers.get("Content-Length")) + if content_length is None: + return None + return already_downloaded + content_length diff --git a/api/routers/optimize.py b/api/routers/optimize.py index db7152c..6081c70 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -1,3 +1,4 @@ +import hashlib import os import re import shutil @@ -11,6 +12,7 @@ _pymeshlab = None _PYMESHLAB_AVAILABLE = False +import numpy as np import trimesh import trimesh.visual from fastapi import APIRouter, HTTPException, UploadFile, File @@ -34,22 +36,38 @@ class SmoothRequest(BaseModel): iterations: int +class TransformRequest(BaseModel): + path: str # format: "{collection}/{filename}" + matrix: list[list[float]] # row-major 4x4 world transform + + def _require_pymeshlab(): if not _PYMESHLAB_AVAILABLE: raise HTTPException(503, "pymeshlab is unavailable on this system (DLL blocked by Windows Application Control policy)") +def _resolve_input_path(raw_path: str) -> Path: + candidate = Path(raw_path) + if candidate.is_absolute(): + resolved = candidate.resolve() + if not resolved.exists(): + raise HTTPException(404, f"File not found: {raw_path}") + return resolved + + resolved = (WORKSPACE_DIR / raw_path).resolve() + if not str(resolved).startswith(str(WORKSPACE_DIR.resolve())): + raise HTTPException(400, "Invalid path") + if not resolved.exists(): + raise HTTPException(404, f"File not found: {raw_path}") + return resolved + + @router.post("/mesh") def optimize_mesh(body: OptimizeRequest): _require_pymeshlab() target_faces = max(100, min(500_000, body.target_faces)) - # Security: prevent path traversal - input_path = (WORKSPACE_DIR / body.path).resolve() - if not str(input_path).startswith(str(WORKSPACE_DIR.resolve())): - raise HTTPException(400, "Invalid path") - if not input_path.exists(): - raise HTTPException(404, f"File not found: {body.path}") + input_path = _resolve_input_path(body.path) tmp_dir = tempfile.mkdtemp() try: @@ -59,21 +77,38 @@ def optimize_mesh(body: OptimizeRequest): stem = input_path.stem output_name = f"{stem}_opt{target_faces}.glb" - output_path = input_path.parent / output_name + output_dir = input_path.parent if str(input_path).startswith(str(WORKSPACE_DIR.resolve())) else WORKSPACE_DIR / "Workflows" + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / output_name result.export(str(output_path)) - # Reconstruct the collection name from the path - collection_name = body.path.split("/")[0] face_count = len(result.faces) - return {"url": f"/workspace/{collection_name}/{output_name}", "face_count": face_count} + rel = output_path.relative_to(WORKSPACE_DIR).as_posix() + return {"url": f"/workspace/{rel}", "face_count": face_count} def _has_texture(geom: trimesh.Trimesh) -> bool: - return ( - isinstance(geom.visual, trimesh.visual.TextureVisuals) - and geom.visual.material is not None - and getattr(geom.visual.material, "image", None) is not None - ) + if not isinstance(geom.visual, trimesh.visual.TextureVisuals): + return False + mat = geom.visual.material + if mat is None: + return False + # Simple material (SimpleMaterial / Material) + if getattr(mat, "image", None) is not None: + return True + # PBR material (from Trellis2 SLaT texturing and GLB imports) + if getattr(mat, "baseColorTexture", None) is not None: + return True + return False + + +def _get_texture_image(geom: trimesh.Trimesh): + """Return the base color texture image regardless of material type.""" + mat = geom.visual.material + img = getattr(mat, "image", None) + if img is not None: + return img + return getattr(mat, "baseColorTexture", None) def _decimate(input_path: str, target_faces: int, tmp_dir: str) -> trimesh.Trimesh: @@ -93,8 +128,8 @@ def _decimate(input_path: str, target_faces: int, tmp_dir: str) -> trimesh.Trime tex_in = os.path.join(tmp_dir, "texture.png") obj_out = os.path.join(tmp_dir, "output.obj") - # Save texture image under a known filename - geom.visual.material.image.save(tex_in) + # Save texture image under a known filename (handles PBR and simple materials) + _get_texture_image(geom).save(tex_in) # Export OBJ (trimesh writes UV coords + MTL) geom.export(obj_in) @@ -146,11 +181,7 @@ def smooth_mesh(body: SmoothRequest): _require_pymeshlab() iterations = max(1, min(20, body.iterations)) - input_path = (WORKSPACE_DIR / body.path).resolve() - if not str(input_path).startswith(str(WORKSPACE_DIR.resolve())): - raise HTTPException(400, "Invalid path") - if not input_path.exists(): - raise HTTPException(404, f"File not found: {body.path}") + input_path = _resolve_input_path(body.path) tmp_dir = tempfile.mkdtemp() try: @@ -160,11 +191,41 @@ def smooth_mesh(body: SmoothRequest): stem = input_path.stem output_name = f"{stem}_smooth{iterations}.glb" - output_path = input_path.parent / output_name + output_dir = input_path.parent if str(input_path).startswith(str(WORKSPACE_DIR.resolve())) else WORKSPACE_DIR / "Workflows" + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / output_name result.export(str(output_path)) - collection_name = body.path.split("/")[0] - return {"url": f"/workspace/{collection_name}/{output_name}"} + rel = output_path.relative_to(WORKSPACE_DIR).as_posix() + return {"url": f"/workspace/{rel}"} + + +@router.post("/transform") +def transform_mesh(body: TransformRequest): + # Bake an interactive-gizmo transform into the GLB at scene level so it + # persists to export. Pure trimesh β€” no pymeshlab needed. + input_path = _resolve_input_path(body.path) + + matrix = np.asarray(body.matrix, dtype=float) + if matrix.shape != (4, 4): + raise HTTPException(400, "matrix must be a 4x4 array") + if not np.all(np.isfinite(matrix)): + raise HTTPException(400, "matrix contains non-finite values") + + # Keep the loaded result as-is (Scene when textured/multi-geometry) so + # apply_transform preserves materials and UVs. + loaded = trimesh.load(str(input_path)) + loaded.apply_transform(matrix) + + stem = input_path.stem + output_name = f"{stem}_xf_{uuid.uuid4().hex[:8]}.glb" + output_dir = input_path.parent if str(input_path).startswith(str(WORKSPACE_DIR.resolve())) else WORKSPACE_DIR / "Workflows" + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / output_name + loaded.export(str(output_path)) + + rel = output_path.relative_to(WORKSPACE_DIR).as_posix() + return {"url": f"/workspace/{rel}"} def _smooth(input_path: str, iterations: int, tmp_dir: str) -> trimesh.Trimesh: @@ -183,7 +244,7 @@ def _smooth(input_path: str, iterations: int, tmp_dir: str) -> trimesh.Trimesh: tex_in = os.path.join(tmp_dir, "texture.png") obj_out = os.path.join(tmp_dir, "output.obj") - geom.visual.material.image.save(tex_in) + _get_texture_image(geom).save(tex_in) geom.export(obj_in) if os.path.exists(mtl_in): @@ -218,6 +279,121 @@ class ImportByPathRequest(BaseModel): path: str # absolute path on disk +# --------------------------------------------------------------------------- +# Gaussian Splatting support +# +# A 3DGS .ply is a point cloud whose colour lives in SH coefficients +# (f_dc_*), not in a texture or vertex colours β€” trimesh strips all of it and +# yields a blank blob. Detect those files by their header and convert them to +# the standard binary .splat format (rowLength = 32) read by the splat viewer: +# pos 3 Γ— float32 (offset 0) +# scale 3 Γ— float32 (offset 12) = exp(scale_i) +# rgba 4 Γ— uint8 (offset 24) = 0.5 + C0Β·f_dc, sigmoid(opacity) +# rot 4 Γ— uint8 (offset 28) = normalised quaternion (w,x,y,z) +# --------------------------------------------------------------------------- + +_SH_C0 = 0.28209479177387814 + +# Bump when _convert_gaussian_ply_to_splat changes, to invalidate cached .splats +# whose source .ply mtime is unchanged. +_SPLAT_CONV_VERSION = 2 + +_PLY_TYPE_MAP = { + "char": "i1", "uchar": "u1", "int8": "i1", "uint8": "u1", + "short": "i2", "ushort": "u2", "int16": "i2", "uint16": "u2", + "int": "i4", "uint": "u4", "int32": "i4", "uint32": "u4", + "float": "f4", "float32": "f4", "double": "f8", "float64": "f8", +} + + +def _is_gaussian_ply(file_path: Path) -> bool: + try: + with open(file_path, "rb") as f: + head = f.read(2048) + except OSError: + return False + if not head.startswith(b"ply"): + return False + text = head.split(b"end_header")[0].decode("ascii", "ignore") + return all(marker in text for marker in ("f_dc_0", "scale_0", "rot_0")) + + +def _convert_gaussian_ply_to_splat(ply_path: Path, out_path: str) -> None: + import numpy as np + + with open(ply_path, "rb") as f: + if f.readline().strip() != b"ply": + raise ValueError("not a ply file") + fmt = None + count = 0 + element = None + props: list[tuple[str, str]] = [] + while True: + line = f.readline() + if not line: + raise ValueError("unexpected EOF in ply header") + parts = line.split() + if not parts: + continue + kw = parts[0] + if kw == b"format": + fmt = parts[1] + elif kw == b"element": + element = parts[1] + if element == b"vertex": + count = int(parts[2]) + elif kw == b"property" and element == b"vertex": + # GS plys use only scalar float properties (no lists) + props.append((parts[2].decode(), parts[1].decode())) + elif kw == b"end_header": + break + if fmt != b"binary_little_endian": + raise ValueError(f"unsupported ply format: {fmt!r}") + + dtype = np.dtype([(n, "<" + _PLY_TYPE_MAP[t]) for n, t in props]) + data = np.frombuffer(f.read(count * dtype.itemsize), dtype=dtype, count=count) + + def col(*names): + return np.stack([data[n].astype(np.float32) for n in names], axis=1) + + xyz = col("x", "y", "z") + scale = np.exp(col("scale_0", "scale_1", "scale_2")) + + # Normalise into the viewer's space: a 3DGS lives in an arbitrary world + # frame (offset origin, any scale) and often has a few far-away "floater" + # splats. Centre x/z on the robust 1–99 percentile box and scale to fit so + # the model always lands in front of the camera (meshes get this for free + # via Box3 centring; splats don't). The gaussian sizes scale with it. + lo = np.percentile(xyz, 1, axis=0) + hi = np.percentile(xyz, 99, axis=0) + center = (lo + hi) / 2.0 + extent = float(np.max(hi - lo)) + factor = (2.0 / extent) if extent > 1e-6 else 1.0 + xyz = (xyz - center) * factor + scale = scale * factor + + # Stand the model on the grid (y = 0), matching the mesh viewer. The client + # flips the splat 180Β° about Z (3DGS is Y-down), negating Y so the robust + # *top* (99th pct) of this space becomes the floor; shift it to 0 β†’ feet on the grid. + xyz[:, 1] -= float(np.percentile(xyz[:, 1], 99)) + rgb = np.clip((0.5 + _SH_C0 * col("f_dc_0", "f_dc_1", "f_dc_2")) * 255.0, 0, 255) + alpha = np.clip(1.0 / (1.0 + np.exp(-data["opacity"].astype(np.float32))) * 255.0, 0, 255) + rot = col("rot_0", "rot_1", "rot_2", "rot_3") + rot /= np.linalg.norm(rot, axis=1, keepdims=True) + rot_u8 = np.clip(rot * 128.0 + 128.0, 0, 255) + + out = np.zeros(count, dtype=[ + ("pos", " dict: + nodes = manifest.get("nodes") or [] + if nodes and model_dir_override: + node_id = Path(model_dir_override).name + return next((n for n in nodes if n.get("id") == node_id), nodes[0]) + if nodes: + return nodes[0] + return {} + + +def _resolve_ready_schema(GenClass, node: dict, manifest: dict) -> list: + try: + return GenClass.params_schema() + except Exception: + return node.get("params_schema") or manifest.get("params_schema", []) + + +def _apply_manifest_metadata(gen, manifest: dict, node: dict) -> None: + gen.hf_repo = node.get("hf_repo") or manifest.get("hf_repo", "") + gen.hf_skip_prefixes = node.get("hf_skip_prefixes") or manifest.get("hf_skip_prefixes", []) + gen.download_check = node.get("download_check") or manifest.get("download_check", "") + gen._params_schema = node.get("params_schema") or manifest.get("params_schema", []) + + # ------------------------------------------------------------------ # # Main loop # ------------------------------------------------------------------ # @@ -97,38 +121,24 @@ def main() -> None: "traceback": traceback.format_exc()}) return - # Announce readiness and send params_schema so ExtensionProcess - # can serve it without needing to query the subprocess later. - # We try to get it from the generator class (may be a classmethod), - # falling back to the manifest field. - try: - schema = GenClass.params_schema() - except Exception: - node0 = (manifest.get("nodes") or [{}])[0] - schema = manifest.get("params_schema", []) or node0.get("params_schema", []) - send({"type": "ready", "params_schema": schema}) - # Support both flat manifest (legacy) and nodes[] format. # Use MODEL_DIR to find the correct node for multi-node extensions: # MODEL_DIR is set by ExtensionProcess to MODELS_DIR/ext_id/node_id, # so its last component matches the node id. - nodes = manifest.get("nodes") or [] - node = {} - if nodes and _MODEL_DIR_OVERRIDE: - node_id = Path(_MODEL_DIR_OVERRIDE).name - node = next((n for n in nodes if n.get("id") == node_id), nodes[0]) - elif nodes: - node = nodes[0] + node = _select_node(manifest, _MODEL_DIR_OVERRIDE) + + # Announce readiness and send params_schema so ExtensionProcess + # can serve it without needing to query the subprocess later. + # We try to get it from the generator class (may be a classmethod), + # falling back to the selected node, then to the top-level manifest. + send({"type": "ready", "params_schema": _resolve_ready_schema(GenClass, node, manifest)}) # Use MODEL_DIR env var (set by ExtensionProcess) when available so the # generator uses the exact same path that is_downloaded() checks against. # Falls back to MODELS_DIR/manifest_id for legacy / standalone use. model_dir = Path(_MODEL_DIR_OVERRIDE) if _MODEL_DIR_OVERRIDE else MODELS_DIR / model_id gen = GenClass(model_dir, WORKSPACE_DIR) - gen.hf_repo = manifest.get("hf_repo", "") or node.get("hf_repo", "") - gen.hf_skip_prefixes = manifest.get("hf_skip_prefixes", []) or node.get("hf_skip_prefixes", []) - gen.download_check = manifest.get("download_check", "") or node.get("download_check", "") - gen._params_schema = manifest.get("params_schema", []) or node.get("params_schema", []) + _apply_manifest_metadata(gen, manifest, node) # Active cancel events keyed by request id _cancel: dict[str, threading.Event] = {} diff --git a/api/services/extension_process.py b/api/services/extension_process.py index 89622a2..60ce678 100644 --- a/api/services/extension_process.py +++ b/api/services/extension_process.py @@ -12,6 +12,7 @@ import os import platform import queue +import re import subprocess import sys import threading @@ -20,6 +21,10 @@ from typing import Callable, Optional _RUNNER_PATH = Path(__file__).parent.parent / "runner.py" +_MISSING_MODULE_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]") +_AUTO_REPAIR_PACKAGE_MAP = { + "PIL": "Pillow", +} def _venv_python(ext_dir: Path) -> Path: @@ -68,8 +73,11 @@ def _build_env(self) -> dict: env["MODELS_DIR"] = str(MODELS_DIR) env["WORKSPACE_DIR"] = str(WORKSPACE_DIR) env["MODLY_API_DIR"] = str(Path(__file__).parent.parent) + if sys.platform == "darwin": + env.setdefault("NUMBA_DISABLE_JIT", "1") # Pass the exact model_dir so runner.py doesn't have to re-derive it # from manifest["id"] (which is the ext_id, not the composite node id). + # runner.py extracts the node id from MODEL_DIR's trailing path component. if self.model_dir is not None: env["MODEL_DIR"] = str(self.model_dir) # Extension venvs are based on python-embed which ships without a CA bundle. @@ -91,53 +99,128 @@ def _start(self) -> None: "Run the extension's setup.py first." ) - self._proc = subprocess.Popen( - [str(python), str(_RUNNER_PATH)], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - bufsize=1, - env=self._build_env(), - ) + for attempt in range(3): + # Use a fresh queue per subprocess lifetime so late messages from an + # older reader thread cannot poison startup for the new process. + run_queue: queue.Queue = queue.Queue() + self._queue = run_queue + self._proc = subprocess.Popen( + [str(python), str(_RUNNER_PATH)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + env=self._build_env(), + ) + + # Background thread: read stdout β†’ queue + reader = threading.Thread(target=self._read_loop, args=(self._proc, run_queue), daemon=True) + reader.start() - # Background thread: read stdout β†’ queue - reader = threading.Thread(target=self._read_loop, daemon=True) - reader.start() + # Background thread: forward stderr to our stderr + stderr_fwd = threading.Thread(target=self._stderr_loop, args=(self._proc,), daemon=True) + stderr_fwd.start() - # Background thread: forward stderr to our stderr - stderr_fwd = threading.Thread(target=self._stderr_loop, daemon=True) - stderr_fwd.start() + # Wait for ready β€” runner sends params_schema in this message + msg = self._recv(timeout=None) + if msg.get("type") == "ready": + # Override params_schema with what the generator class actually declares + if msg.get("params_schema"): + self._params_schema = msg["params_schema"] + + print(f"[ExtensionProcess] {self.MODEL_ID} subprocess started (pid {self._proc.pid})") + return - # Wait for ready β€” runner sends params_schema in this message - msg = self._recv(timeout=None) - if msg.get("type") != "ready": self._proc.kill() - raise RuntimeError(f"[{self.MODEL_ID}] Expected 'ready', got: {msg}") + self._proc.wait() + missing_module = self._extract_missing_module(msg) + package_name = self._resolve_auto_repair_package(missing_module) if missing_module else None + if package_name and attempt < 2: + self._install_missing_package(python, missing_module, package_name) + continue - # Override params_schema with what the generator class actually declares - if msg.get("params_schema"): - self._params_schema = msg["params_schema"] + raise RuntimeError(f"[{self.MODEL_ID}] Expected 'ready', got: {msg}") - print(f"[ExtensionProcess] {self.MODEL_ID} subprocess started (pid {self._proc.pid})") + def _extract_missing_module(self, msg: dict) -> Optional[str]: + """Returns missing import name from a runner error payload, if present.""" + blob = f"{msg.get('message', '')}\n{msg.get('traceback', '')}" + match = _MISSING_MODULE_RE.search(blob) + return match.group(1) if match else None + + def _resolve_auto_repair_package(self, module_name: str) -> Optional[str]: + """ + Maps a missing import name to a pip package for safe auto-repair. + + Important: do not guess package names for arbitrary missing modules, + because that can install wrong packages and break environments. + """ + if module_name in _AUTO_REPAIR_PACKAGE_MAP: + return _AUTO_REPAIR_PACKAGE_MAP[module_name] + root = module_name.split(".")[0] + return _AUTO_REPAIR_PACKAGE_MAP.get(root) + + def _install_missing_package(self, python: Path, module_name: str, package_name: str) -> None: + """Best-effort auto-repair for a known missing import in extension venv.""" + print( + f"[ExtensionProcess] {self.MODEL_ID} missing module '{module_name}' " + f"-> installing '{package_name}'" + ) + try: + subprocess.run( + [str(python), "-m", "pip", "install", package_name], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError as exc: + details = (exc.stderr or exc.stdout or "").strip() + raise RuntimeError( + f"[{self.MODEL_ID}] Auto-repair failed while installing '{package_name}' " + f"for missing module '{module_name}'.\n{details[-2000:]}" + ) from exc - def _read_loop(self) -> None: + def _read_loop(self, proc: subprocess.Popen, msg_queue: queue.Queue) -> None: """Continuously reads stdout and pushes parsed JSON to the queue.""" try: - for line in self._proc.stdout: + for line in proc.stdout: line = line.strip() if line: try: - self._queue.put(json.loads(line)) + msg_queue.put(json.loads(line)) except json.JSONDecodeError: - print(f"[{self.MODEL_ID}] bad JSON: {line}", file=sys.stderr) + print(f"[{self.MODEL_ID}] {line}", file=sys.stderr) finally: - self._queue.put(None) # sentinel: process is done - - def _stderr_loop(self) -> None: - """Forwards subprocess stderr to the main process stderr.""" - for line in self._proc.stderr: - print(f"[{self.MODEL_ID}] {line}", end="", file=sys.stderr) + msg_queue.put(None) # sentinel: process is done + + def _stderr_loop(self, proc: subprocess.Popen) -> None: + """Forward subprocess stderr to the main process stderr, emitting + one line every time we see EITHER '\\n' or '\\r'. tqdm writes live + progress updates with '\\r' only, so a newline-only iterator would + buffer every tick until the loop exits with '\\n' β€” which is why + the HUD's log pane went dark during multi-minute volume decode. + + No per-line extension-id prefix: the HUD log pane is a single + truncated line, and eating 20 characters with "[modly-hy3d2-mac] " + hides the tail of the tqdm bar the user actually wants to read. + """ + stream = proc.stderr + if stream is None: + return + buf = [] + while True: + ch = stream.read(1) + if not ch: + if buf: + print(''.join(buf), file=sys.stderr, flush=True) + return + if ch in ("\r", "\n"): + if buf: + print(''.join(buf), file=sys.stderr, flush=True) + buf = [] + else: + buf.append(ch) def _send(self, msg: dict) -> None: with self._lock: @@ -174,7 +257,7 @@ def load(self) -> None: self._send({"action": "load"}) msg = self._recv(timeout=None) # model load can be arbitrarily slow - if msg.get("type") == "loaded": + if msg.get('type') in ['loaded', 'ready']: self._loaded = True elif msg.get("type") == "error": raise RuntimeError(msg.get("traceback") or msg.get("message")) @@ -208,14 +291,45 @@ def generate( "outputs_dir": str(self.outputs_dir) if self.outputs_dir else None, }) + # Grace period after sending a cooperative cancel before hard-killing + # the subprocess. Long enough to let generators that check cancel_event + # between steps shut down cleanly, short enough that the user isn't + # left staring at a stuck UI when the subprocess is blocked inside a + # native call (octree decode, marching cubes, etc.) that ignores stdin. + CANCEL_GRACE_SECONDS = 3.0 + + cancel_sent_at: Optional[float] = None while True: # Check for cancellation if cancel_event and cancel_event.is_set(): - self._send({"action": "cancel", "id": req_id}) - # Drain until the subprocess acknowledges - while True: - msg = self._recv(timeout=30.0) - if msg.get("type") in ("cancelled", "done", "error"): + if cancel_sent_at is None: + # First observation of the cancel β€” ask the subprocess to stop. + try: + self._send({"action": "cancel", "id": req_id}) + except Exception: + pass + import time + cancel_sent_at = time.monotonic() + else: + import time + if time.monotonic() - cancel_sent_at >= CANCEL_GRACE_SECONDS: + # Grace period expired β€” the subprocess is not + # responding (almost certainly stuck in native code). + # Hard-kill it and drop our state so the next + # generation forces a fresh load. + try: + if self._proc and self._proc.poll() is None: + self._proc.kill() + self._proc.wait(timeout=5.0) + except Exception: + pass + self._loaded = False + self._proc = None + print( + f"[ExtensionProcess] {self.MODEL_ID} subprocess killed " + f"after {CANCEL_GRACE_SECONDS}s grace; model will reload on next run", + file=sys.stderr, + ) raise GenerationCancelled() # Poll queue with short timeout so we can re-check cancel_event @@ -249,12 +363,28 @@ def params_schema(self) -> list: return self._params_schema def stop(self) -> None: - """Gracefully shut down the subprocess.""" - if self._proc and self._proc.poll() is None: + """Hard-stop the subprocess. + + Used by Free Memory / unload_all. Cooperative shutdown was the wrong + semantics here: torch.mps.empty_cache() does not reliably release + wired Metal pages, so only process exit actually returns the memory + to the OS. We SIGKILL, reap the zombie, and drop our refs so the + next load() starts a fresh subprocess. + """ + proc = self._proc + self._proc = None + self._loaded = False + if proc and proc.poll() is None: try: - self._send({"action": "shutdown"}) - self._proc.wait(timeout=15) + proc.kill() + proc.wait(timeout=5) except Exception: - self._proc.kill() - self._loaded = False - self._proc = None + pass + self._drain_queue() + + def _drain_queue(self) -> None: + while not self._queue.empty(): + try: + self._queue.get_nowait() + except queue.Empty: + break diff --git a/api/services/generator_registry.py b/api/services/generator_registry.py index de1f20c..b36c6c9 100644 --- a/api/services/generator_registry.py +++ b/api/services/generator_registry.py @@ -106,7 +106,8 @@ def _discover_extensions() -> Dict[str, Tuple[type, dict]]: "hf_repo": node.get("hf_repo", ""), "download_check": node.get("download_check", ""), "hf_skip_prefixes": node.get("hf_skip_prefixes", []), - "params_schema": node.get("params_schema", []), + "hf_include_prefixes": node.get("hf_include_prefixes", []), + "params_schema": node.get("params_schema", manifest.get("params_schema", [])), "input": node.get("input", "image"), "output": node.get("output", "mesh"), } diff --git a/api/tests/test_extension_process.py b/api/tests/test_extension_process.py new file mode 100644 index 0000000..cbe7562 --- /dev/null +++ b/api/tests/test_extension_process.py @@ -0,0 +1,25 @@ +import io +import queue +import unittest + +from services.extension_process import ExtensionProcess + + +class ExtensionProcessTests(unittest.TestCase): + def test_read_loop_writes_sentinel_to_own_queue_only(self) -> None: + proc = ExtensionProcess(ext_dir=None, manifest={"id": "demo"}) # type: ignore[arg-type] + + old_queue: queue.Queue = queue.Queue() + new_queue: queue.Queue = queue.Queue() + proc._queue = new_queue + + fake_proc = type("FakeProc", (), {"stdout": io.StringIO("")})() + + proc._read_loop(fake_proc, old_queue) + + self.assertFalse(old_queue.empty()) + self.assertTrue(new_queue.empty()) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_runner.py b/api/tests/test_runner.py new file mode 100644 index 0000000..ada9dfc --- /dev/null +++ b/api/tests/test_runner.py @@ -0,0 +1,68 @@ +import unittest +import os +import tempfile +import importlib +from pathlib import Path + + +_tmp_ext_dir = tempfile.mkdtemp(prefix="modly-runner-test-") +Path(_tmp_ext_dir, "manifest.json").write_text("{}", encoding="utf-8") +os.environ.setdefault("EXTENSION_DIR", _tmp_ext_dir) + +runner = importlib.import_module("runner") +_apply_manifest_metadata = runner._apply_manifest_metadata +_resolve_ready_schema = runner._resolve_ready_schema +_select_node = runner._select_node + + +class RunnerTests(unittest.TestCase): + def test_select_node_uses_model_dir_override(self) -> None: + manifest = { + "nodes": [ + {"id": "fast", "params_schema": [{"id": "a"}]}, + {"id": "quality", "params_schema": [{"id": "b"}]}, + ] + } + + node = _select_node(manifest, str(Path("/tmp/ext/quality"))) + + self.assertEqual(node["id"], "quality") + + def test_ready_schema_falls_back_to_selected_node_schema(self) -> None: + class GenClass: + @classmethod + def params_schema(cls): + raise RuntimeError("not available") + + manifest = {"params_schema": [{"id": "manifest"}]} + node = {"params_schema": [{"id": "node"}]} + + schema = _resolve_ready_schema(GenClass, node, manifest) + + self.assertEqual(schema, [{"id": "node"}]) + + def test_apply_manifest_metadata_prefers_node_specific_values(self) -> None: + gen = type("Gen", (), {})() + manifest = { + "hf_repo": "top/repo", + "hf_skip_prefixes": ["top/"], + "download_check": "top/file", + "params_schema": [{"id": "top"}], + } + node = { + "hf_repo": "node/repo", + "hf_skip_prefixes": ["node/"], + "download_check": "node/file", + "params_schema": [{"id": "node"}], + } + + _apply_manifest_metadata(gen, manifest, node) + + self.assertEqual(gen.hf_repo, "node/repo") + self.assertEqual(gen.hf_skip_prefixes, ["node/"]) + self.assertEqual(gen.download_check, "node/file") + self.assertEqual(gen._params_schema, [{"id": "node"}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/texture_baker/setup.py b/api/texture_baker/setup.py index 1b45cf2..e5158cb 100644 --- a/api/texture_baker/setup.py +++ b/api/texture_baker/setup.py @@ -14,6 +14,7 @@ library_name = "texture_baker" IS_WINDOWS = platform.system() == "Windows" +IS_MACOS = platform.system() == "Darwin" def get_extensions(): @@ -39,6 +40,18 @@ def get_extensions(): if debug_mode: cxx_flags += ["/Z7"] extra_link_args += ["/DEBUG"] + elif IS_MACOS: + # Prefer a conservative Apple Silicon toolchain over OpenMP-specific flags. + cxx_flags = [ + "-O3" if not debug_mode else "-O0", + "-fdiagnostics-color=always", + "-mmacosx-version-min=11.0", + ] + if use_native_arch: + cxx_flags.append("-mcpu=apple-m1") + if debug_mode: + cxx_flags += ["-g", "-UNDEBUG"] + extra_link_args += ["-O0", "-g"] else: # GCC/Clang flags cxx_flags = [ @@ -91,8 +104,17 @@ def get_extensions(): sources += glob.glob( os.path.join(this_dir, library_name, "csrc", "**", "*.mm"), recursive=True ) - extra_compile_args.update({"cxx": ["-O3", "-arch", "arm64", "-mmacosx-version-min=10.15"]}) - extra_link_args += ["-arch", "arm64"] + if IS_MACOS: + if "-arch" not in extra_link_args: + extra_link_args += [ + "-arch", + "arm64", + "-framework", + "Metal", + "-framework", + "Foundation", + ] + cxx_flags.extend(["-arch", "arm64"]) extensions.append( extension( diff --git a/api/uv_unwrapper/setup.py b/api/uv_unwrapper/setup.py index bff9eec..8284ed9 100644 --- a/api/uv_unwrapper/setup.py +++ b/api/uv_unwrapper/setup.py @@ -12,6 +12,7 @@ library_name = "uv_unwrapper" IS_WINDOWS = platform.system() == "Windows" +IS_MACOS = platform.system() == "Darwin" def get_extensions(): @@ -19,8 +20,7 @@ def get_extensions(): if debug_mode: print("Compiling in debug mode") - is_mac = True if torch.backends.mps.is_available() else False - use_native_arch = not is_mac and not IS_WINDOWS and os.getenv("USE_NATIVE_ARCH", "1") == "1" + use_native_arch = not IS_MACOS and not IS_WINDOWS and os.getenv("USE_NATIVE_ARCH", "1") == "1" extension = CppExtension extra_link_args = [] @@ -31,12 +31,13 @@ def get_extensions(): if debug_mode: cxx_flags += ["/Z7", "/UNDEBUG"] extra_link_args += ["-O0"] - elif is_mac: + elif IS_MACOS: cxx_flags = [ "-O3" if not debug_mode else "-O0", "-fdiagnostics-color=always", - "-Xclang -fopenmp", - "-mmacosx-version-min=10.15", + "-mmacosx-version-min=11.0", + "-arch", + "arm64", ] if debug_mode: cxx_flags += ["-g", "-UNDEBUG"] @@ -74,9 +75,7 @@ def get_extensions(): define_macros=define_macros, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, - libraries=["c10", "torch", "torch_cpu", "torch_python"] + ["omp"] - if is_mac - else [], + libraries=["c10", "torch", "torch_cpu", "torch_python"] if IS_MACOS else [], ) ) diff --git a/arch/decisions/APPLE-SILICON-SUPPORT.md b/arch/decisions/APPLE-SILICON-SUPPORT.md new file mode 100644 index 0000000..c49c153 --- /dev/null +++ b/arch/decisions/APPLE-SILICON-SUPPORT.md @@ -0,0 +1,114 @@ +# APPLE-SILICON-SUPPORT + +- Status: proposed +- Date: 2026-04-23 + +## Decision + +Modly supports macOS on Apple Silicon (`darwin/arm64`) as a first-class +platform. This ADR consolidates the runtime, packaging, extension, and +workflow rules needed to run the image-to-mesh pipeline reliably on 16 GB +unified-memory Macs. + +Scope and operating rules: + +- macOS support targets Apple Silicon only. See `package.json:99`. +- Intel macOS, universal binaries, and Rosetta fallback are out of scope. +- Model weights stay separate from extension code and are installed per node. See `api/routers/model.py:76` and `api/runner.py:101`. +- The Mac workflow is sequential and memory-budgeted: one heavy generative + stage resident at a time. See `src/areas/workflows/workflowRunStore.ts:185`. +- Download/install state must be observable and resumable. See `electron/main/model-downloader.ts:116`. +- Releasing GPU memory means terminating the owning subprocess. See `api/services/extension_process.py:205` and `electron/main/index.ts:105`. +- Generation progress and cancel behavior must remain visible and responsive in + the UI. See `api/services/extension_process.py:135` and `src/areas/workflows/workflowRunStore.ts:232`. + +## Context + +Apple Silicon changes the constraints under which Modly runs: + +- Unified memory means overlapping heavy GPU stages can destabilize the whole + machine on 16 GB systems. +- Metal/MPS memory is not returned predictably by Python-side cleanup alone; + process exit is the reliable release boundary. +- Large model downloads need byte-level visibility, stall detection, and proper + resume behavior to avoid appearing hung or silently reinstalling from zero. +- Extension manifests now need per-node distribution metadata because one + extension can expose multiple model variants that share code but differ in + weights, defaults, and required artifacts. +- Workflow graphs need preflight validation before execution so invalid wiring + is reported without replacing the current mesh view with a terminal error + state. +- The renderer needs progress text that stays live through long native phases + and a cancel path that clears UI state immediately even if backend teardown + takes longer. + +## Consequences + +- Packaging: + Modly packages macOS as an Apple Silicon build path only, including the + embedded Python runtime in the app bundle. See `package.json:99`. + +- Extension and model distribution: + Extension payloads contain code, manifests, setup scripts, and lightweight + assets. Model nodes declare their own `download_check`, can narrow downloads + with `hf_include_prefixes` and `hf_skip_prefixes`, and may provide + node-specific `params_schema` and `param_defaults` with top-level fallback. + See `api/routers/model.py:76`, `api/runner.py:84`, and + `electron/main/model-downloader.ts:116`. + +- Runtime selection and defaults: + The runner resolves the active node from `MODEL_DIR` so multi-node extensions + use the correct schema, model directory, and node-specific metadata. Workflow + submission merges displayed parameter defaults under user overrides before the + request reaches Python. See `api/runner.py:84` and + `src/areas/workflows/workflowRunStore.ts:207`. + +- Mesh optimization path handling: + Smooth and decimate operations accept both workspace-relative meshes and + imported absolute-path meshes, then write optimized output back into the + workspace so the result remains visible and reusable in the app. See + `api/routers/optimize.py:42` and `src/areas/generate/GeneratePage.tsx:331`. + +- Memory-budgeted workflow: + Heavy stages hand off through files and unload before the next heavy stage + begins. CPU-oriented stages can run between GPU-heavy stages without + competing for MPS residency. See `electron/main/index.ts:105`, + `api/services/extension_process.py:214`, and + `src/areas/workflows/workflowRunStore.ts:142`. + +- Download behavior: + The downloader emits byte-level progress, file context, and stall state. + Partial downloads are preserved as `.part` files. Resume is attempted against + the resolved final URL so `Range` works even when upstream redirects to a CDN. + Install completion is verified by the declared `download_check`, not by + directory existence alone. See `electron/main/model-downloader.ts:10`, + `electron/main/model-downloader.ts:31`, and `api/routers/model.py:84`. + +- Subprocess lifecycle: + Extension subprocesses are owned as a full process tree. On Unix, the Python + bridge runs as its own process-group leader and Modly kills the process group + on quit. Free-memory/unload operations hard-stop the subprocess. Cancel first + sends a cooperative request, then escalates to a kill after a short grace + period if native code is still blocking. See `api/services/extension_process.py:78` + and `electron/main/index.ts:112`. + +- Observability and UX: + Generator stderr stays available for tqdm-style progress parsing, long phases + surface readable status text, and cancel clears renderer job state + immediately. Workflow editors run a preflight pass before execution and + surface wiring problems through inline warnings and toasts instead of + replacing the current mesh view. Error output in the HUD remains + copyable/selectable. The top bar includes a live RAM indicator backed by a + main-process `system:memory` IPC call; macOS uses `vm_stat` to approximate + Activity Monitor's "Memory Used" semantics and other platforms fall back to + `total - free`. macOS uses native window controls instead of custom + right-side controls. See + `api/services/extension_process.py:135`, + `src/areas/workflows/preflight.ts:51`, + `src/areas/generate/components/WorkflowPanel.tsx:425`, + `src/areas/workflows/WorkflowsPage.tsx:847`, + `src/shared/components/ui/Toast.tsx:4`, + `electron/main/ipc-handlers.ts:372`, + `src/shared/components/layout/MemoryIndicator.tsx:9`, + `src/shared/components/layout/TopBar.tsx:4`, and + `src/areas/setup/FirstRunSetup.tsx:258`. diff --git a/arch/decisions/README.md b/arch/decisions/README.md new file mode 100644 index 0000000..063df57 --- /dev/null +++ b/arch/decisions/README.md @@ -0,0 +1,10 @@ +# Architecture Decisions + +This directory stores architecture decision records for Modly. + +The current Apple Silicon support work is documented in one consolidated ADR so +the platform scope, runtime assumptions, and operational constraints stay in a +single reviewable document. + +Current ADRs: +- [APPLE-SILICON-SUPPORT](./APPLE-SILICON-SUPPORT.md) diff --git a/electron/main/artifact-registry-service.test.ts b/electron/main/artifact-registry-service.test.ts new file mode 100644 index 0000000..91ba259 --- /dev/null +++ b/electron/main/artifact-registry-service.test.ts @@ -0,0 +1,230 @@ +import assert from 'node:assert/strict' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import test from 'node:test' + +import { + classifyAssetLibraryCandidate, + listWorkspaceAssetLibrary, + normalizeWorkspaceAssetPath, + openWorkspaceAssetLibraryEntry, + readWorkspaceAssetLibraryEntry, + registerWorkspaceAssetLibraryIpcHandlers, +} from './artifact-registry-service.ts' + +async function withWorkspace(run: (workspaceDir: string) => Promise) { + const workspaceDir = await mkdtemp(path.join(tmpdir(), 'modly-library-')) + try { + await run(workspaceDir) + } finally { + await rm(workspaceDir, { recursive: true, force: true }) + } +} + +test('normalizes only workspace-relative paths under allowed Workflows and Exports roots', () => withWorkspace(async (workspaceDir) => { + assert.equal(normalizeWorkspaceAssetPath(workspaceDir, 'Workflows/checkpoints/hero.glb').workspacePath, 'Workflows/checkpoints/hero.glb') + assert.equal(normalizeWorkspaceAssetPath(workspaceDir, 'Exports/hero.glb').workspacePath, 'Exports/hero.glb') + assert.throws(() => normalizeWorkspaceAssetPath(workspaceDir, '../secret.glb'), /traversal|escape|relative/i) + assert.throws(() => normalizeWorkspaceAssetPath(workspaceDir, '/tmp/secret.glb'), /absolute/i) + assert.throws(() => normalizeWorkspaceAssetPath(workspaceDir, 'Workflows/%2e%2e/secret.glb'), /encoded/i) + assert.throws(() => normalizeWorkspaceAssetPath(workspaceDir, 'Collections/hero.glb'), /allowed workspace library roots/i) +})) + +test('classifies supported assets by capability instead of extension category', () => { + assert.deepEqual(classifyAssetLibraryCandidate({ workspacePath: 'Workflows/a.glb' }), { + capability: 'mesh', state: 'ready', previewKind: '3d-model', openable: true, + }) + assert.deepEqual(classifyAssetLibraryCandidate({ workspacePath: 'Workflows/rigged.gltf', hasRigMetadata: true }), { + capability: 'rigged-mesh', state: 'ready', previewKind: '3d-model', openable: true, + }) + assert.equal(classifyAssetLibraryCandidate({ workspacePath: 'Workflows/motion.bvh' }).capability, 'animation-motion') + assert.equal(classifyAssetLibraryCandidate({ workspacePath: 'Workflows/scan.splat' }).openable, false) + assert.equal(classifyAssetLibraryCandidate({ workspacePath: 'Workflows/notes.txt' }).state, 'unsupported') +}) + +test('lists Workflows and Exports assets while skipping hidden, cache, and internal files', () => withWorkspace(async (workspaceDir) => { + await mkdir(path.join(workspaceDir, 'Workflows/checkpoints'), { recursive: true }) + await mkdir(path.join(workspaceDir, 'Workflows/.hidden'), { recursive: true }) + await mkdir(path.join(workspaceDir, 'Workflows/cache'), { recursive: true }) + await mkdir(path.join(workspaceDir, 'Exports'), { recursive: true }) + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.glb'), 'glb') + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.rigmeta.json'), '{}') + await writeFile(path.join(workspaceDir, 'Workflows/.hidden/private.glb'), 'glb') + await writeFile(path.join(workspaceDir, 'Workflows/cache/temp.glb'), 'glb') + await writeFile(path.join(workspaceDir, 'Exports/exported.ply'), 'ply') + + const result = await listWorkspaceAssetLibrary({ workspaceDir }) + assert.equal(result.success, true) + assert.deepEqual(result.success && result.entries.map((entry) => entry.workspacePath), [ + 'Exports/exported.ply', + 'Workflows/checkpoints/hero.glb', + ]) + assert.equal(result.success && result.entries.find((entry) => entry.workspacePath.endsWith('hero.glb'))?.capability, 'rigged-mesh') + assert.equal(result.success && result.entries.find((entry) => entry.workspacePath.endsWith('exported.ply'))?.openable, false) +})) + +test('reads and opens only safe GLB/GLTF workspace assets', () => withWorkspace(async (workspaceDir) => { + await mkdir(path.join(workspaceDir, 'Workflows/checkpoints'), { recursive: true }) + await mkdir(path.join(workspaceDir, 'Exports'), { recursive: true }) + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.glb'), 'glb') + await writeFile(path.join(workspaceDir, 'Exports/static.ply'), 'ply') + + const read = await readWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath: 'Workflows/checkpoints/hero.glb' }) + assert.equal(read.success, true) + assert.equal(read.success && read.preview.kind, '3d-model') + const opened = await openWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath: 'Workflows/checkpoints/hero.glb' }) + assert.equal(opened.success, true) + assert.equal(opened.success && opened.entry.openable, true) + const blocked = await openWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath: 'Exports/static.ply' }) + assert.equal(blocked.success, false) + assert.equal(!blocked.success && blocked.error.code, 'not-openable') + const unsafe = await readWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath: '../secret.glb' }) + assert.equal(unsafe.success, false) + assert.equal(!unsafe.success && unsafe.error.code, 'unsafe-path') +})) + +test('Electron read/open boundary rejects Windows absolute and UNC workspace paths', () => withWorkspace(async (workspaceDir) => { + await mkdir(path.join(workspaceDir, 'Workflows/checkpoints'), { recursive: true }) + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.glb'), 'glb') + + const unsafeWorkspacePaths = [ + 'C:\\Users\\x\\asset.glb', + 'C:/Users/x/asset.glb', + '\\\\server\\share\\asset.glb', + ] + + for (const workspacePath of unsafeWorkspacePaths) { + const read = await readWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath }) + const opened = await openWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath }) + + assert.equal(read.success, false, `${workspacePath} should be rejected for read`) + assert.equal(!read.success && read.error.code, 'unsafe-path') + assert.equal(opened.success, false, `${workspacePath} should be rejected for open`) + assert.equal(!opened.success && opened.error.code, 'unsafe-path') + } +})) + +test('Electron read/open boundary rejects Windows absolute and UNC sourceWorkspacePath values', () => withWorkspace(async (workspaceDir) => { + await mkdir(path.join(workspaceDir, 'Workflows/checkpoints'), { recursive: true }) + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.glb'), 'glb') + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.landmarks.v1.json'), JSON.stringify({ + sourceWorkspacePath: 'Workflows/checkpoints/hero.glb', + })) + + const unsafeSourcePaths = [ + 'C:\\Users\\x\\asset.glb', + 'C:/Users/x/asset.glb', + '\\\\server\\share\\asset.glb', + ] + + for (const sourceWorkspacePath of unsafeSourcePaths) { + const read = await readWorkspaceAssetLibraryEntry({ + workspaceDir, + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + sourceWorkspacePath, + }) + const opened = await openWorkspaceAssetLibraryEntry({ + workspaceDir, + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + sourceWorkspacePath, + }) + + assert.equal(read.success, false, `${sourceWorkspacePath} should be rejected as read source`) + assert.equal(!read.success && read.error.code, 'unsafe-path') + assert.equal(opened.success, false, `${sourceWorkspacePath} should be rejected as open source`) + assert.equal(!opened.success && opened.error.code, 'unsafe-path') + } +})) + +test('enriches sidecars with safe source, manifest, artifact, version, and provenance metadata', () => withWorkspace(async (workspaceDir) => { + await mkdir(path.join(workspaceDir, 'Workflows/checkpoints'), { recursive: true }) + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.glb'), 'glb') + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.scene.json'), JSON.stringify({ schema: 'scene-manifest' })) + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.landmarks.v1.json'), JSON.stringify({ + sourceWorkspacePath: 'Workflows/checkpoints/hero.glb', + manifestWorkspacePath: 'Workflows/checkpoints/hero.scene.json', + artifactId: 'artifact-hero', + versionId: 'version-1', + provenance: { workflowId: 'wf-1', workflowNodeId: 'node-1' }, + })) + + const read = await readWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json' }) + + assert.equal(read.success, true) + if (!read.success) return + assert.equal(read.entry.source?.workspacePath, 'Workflows/checkpoints/hero.glb') + assert.equal(read.entry.manifest?.workspacePath, 'Workflows/checkpoints/hero.scene.json') + assert.equal(read.entry.manifest?.capability, 'scene-manifest') + assert.equal(read.entry.artifactId, 'artifact-hero') + assert.equal(read.entry.versionId, 'version-1') + assert.equal(read.entry.provenance?.workflowId, 'wf-1') +})) + +test('fails closed for unsafe, self, missing, and mismatched sourceWorkspacePath opens', () => withWorkspace(async (workspaceDir) => { + await mkdir(path.join(workspaceDir, 'Workflows/checkpoints'), { recursive: true }) + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.glb'), 'glb') + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/other.glb'), 'glb') + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/hero.landmarks.v1.json'), JSON.stringify({ + sourceWorkspacePath: 'Workflows/checkpoints/hero.glb', + })) + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/self.landmarks.v1.json'), JSON.stringify({ + sourceWorkspacePath: 'Workflows/checkpoints/self.landmarks.v1.json', + })) + await writeFile(path.join(workspaceDir, 'Workflows/checkpoints/missing.landmarks.v1.json'), JSON.stringify({ + sourceWorkspacePath: 'Workflows/checkpoints/missing.glb', + })) + + const opened = await openWorkspaceAssetLibraryEntry({ + workspaceDir, + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + sourceWorkspacePath: 'Workflows/checkpoints/hero.glb', + }) + assert.equal(opened.success, true) + assert.equal(opened.success && opened.entry.source?.workspacePath, 'Workflows/checkpoints/hero.glb') + + const unsafe = await openWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', sourceWorkspacePath: '../secret.glb' }) + const self = await openWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath: 'Workflows/checkpoints/self.landmarks.v1.json', sourceWorkspacePath: 'Workflows/checkpoints/self.landmarks.v1.json' }) + const missing = await openWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath: 'Workflows/checkpoints/missing.landmarks.v1.json', sourceWorkspacePath: 'Workflows/checkpoints/missing.glb' }) + const mismatched = await openWorkspaceAssetLibraryEntry({ workspaceDir, workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', sourceWorkspacePath: 'Workflows/checkpoints/other.glb' }) + + assert.equal(unsafe.success, false) + assert.equal(!unsafe.success && unsafe.error.code, 'unsafe-path') + assert.equal(self.success, false) + assert.equal(!self.success && self.error.code, 'not-openable') + assert.equal(missing.success, false) + assert.equal(!missing.success && missing.error.code, 'not-openable') + assert.equal(mismatched.success, false) + assert.equal(!mismatched.success && mismatched.error.code, 'not-openable') +})) + +test('IPC read and open handlers forward sourceWorkspacePath without trusting malformed payloads', async () => { + const handlers = new Map Promise>() + registerWorkspaceAssetLibraryIpcHandlers({ + ipcMain: { handle: (channel, handler) => handlers.set(channel, handler) }, + getWorkspaceDir: () => '/tmp/modly-workspace', + }) + + const result = await handlers.get('workspace:library:open')?.({}, { + workspacePath: 'Workflows/hero.landmarks.v1.json', + sourceWorkspacePath: '../secret.glb', + }) + + assert.equal(typeof (result as { success?: unknown }).success, 'boolean') + assert.equal((result as { success: boolean, error?: { code: string } }).error?.code, 'unsafe-path') +}) + +test('registers workspace library IPC handlers with structured results', async () => { + const handlers = new Map Promise>() + registerWorkspaceAssetLibraryIpcHandlers({ + ipcMain: { handle: (channel, handler) => handlers.set(channel, handler) }, + getWorkspaceDir: () => '/tmp/modly-workspace', + }) + + assert.equal(typeof handlers.get('workspace:library:list'), 'function') + assert.equal(typeof handlers.get('workspace:library:read'), 'function') + assert.equal(typeof handlers.get('workspace:library:open'), 'function') + const result = await handlers.get('workspace:library:read')?.({}, { workspacePath: '../escape.glb' }) + assert.equal(typeof (result as { success?: unknown }).success, 'boolean') + assert.equal((result as { success: boolean, error?: { code: string } }).error?.code, 'unsafe-path') +}) diff --git a/electron/main/artifact-registry-service.ts b/electron/main/artifact-registry-service.ts new file mode 100644 index 0000000..3c26de7 --- /dev/null +++ b/electron/main/artifact-registry-service.ts @@ -0,0 +1,366 @@ +import { readdir, readFile, stat } from 'node:fs/promises' +import { basename, extname, isAbsolute, join, relative, resolve, sep } from 'node:path' + +import type { + AssetCapability, + AssetEntryState, + AssetLibraryEntry, + AssetLibraryError, + AssetLibraryListResult, + AssetLibraryOpenResult, + AssetLibraryPreviewKind, + AssetLibraryPreviewPayload, + AssetLibraryReadResult, + AssetLibrarySourceScope, +} from '../../src/shared/types/assetLibrary' +import type { ArtifactProvenance } from '../../src/shared/types/artifacts' + +const WINDOWS_ABSOLUTE_PATH = /^[a-zA-Z]:[\\/]/ +const ENCODED_ESCAPE_PATTERN = /%2e|%2f|%5c/i +const ALLOWED_ROOTS = ['Workflows', 'Exports'] as const +const SKIPPED_DIRS = new Set(['tmp', 'temp', 'cache']) +const INTERNAL_SUFFIXES = ['.artifact.json', '.rigmeta.json'] as const +const TEXT_EXTENSIONS = new Set(['json', 'txt', 'md']) +const INTRINSIC_MOTION_EXTENSIONS = new Set(['bvh', 'npz']) +const MESH_EXTENSIONS = new Set(['glb', 'gltf', 'obj', 'stl', 'ply', 'splat']) + +export interface AssetLibraryClassificationCandidate { + workspacePath: string + hasRigMetadata?: boolean +} + +export interface AssetLibraryClassification { + capability?: AssetCapability + state: AssetEntryState + previewKind: AssetLibraryPreviewKind + openable: boolean + nonOpenableReason?: string +} + +export interface NormalizedWorkspaceAssetPath { + workspacePath: string + absolutePath: string +} + +export interface WorkspaceAssetLibraryRequest { + workspaceDir: string +} + +export interface WorkspaceAssetLibraryReadRequest extends WorkspaceAssetLibraryRequest { + workspacePath: string + sourceWorkspacePath?: string +} + +interface AssetLibraryMetadata { + sourceWorkspacePath?: string + manifestWorkspacePath?: string + artifactId?: string + versionId?: string + provenance?: ArtifactProvenance + warnings: string[] +} + +export interface IpcMainLike { + handle(channel: string, handler: (event: unknown, payload?: unknown) => Promise): void +} + +export interface WorkspaceAssetLibraryIpcDeps { + ipcMain: IpcMainLike + getWorkspaceDir: () => string +} + +function libraryError(code: AssetLibraryError['code'], message: string): AssetLibraryError { + return { code, message } +} + +function normalizeSeparators(input: string): string { + return input.replace(/\\/g, '/') +} + +function isWindowsAbsolutePath(input: string): boolean { + return WINDOWS_ABSOLUTE_PATH.test(input) || input.startsWith('\\\\') +} + +function assertSafeWorkspaceRelativePath(workspacePath: string): string { + const normalized = normalizeSeparators(workspacePath.trim()) + if (!normalized || normalized === '.') throw new Error('Workspace library path must be workspace-relative and non-empty') + if (ENCODED_ESCAPE_PATTERN.test(normalized)) throw new Error('Workspace library path must not contain encoded path escapes') + if (isAbsolute(normalized) || isWindowsAbsolutePath(workspacePath)) throw new Error('Workspace library path must not be absolute') + const segments = normalized.split('/').filter((segment) => segment && segment !== '.') + if (segments.some((segment) => segment === '..')) throw new Error('Workspace library path must not contain traversal segments') + if (!ALLOWED_ROOTS.includes(segments[0] as typeof ALLOWED_ROOTS[number])) { + throw new Error('Workspace library path must stay under allowed workspace library roots') + } + return segments.join('/') +} + +export function normalizeWorkspaceAssetPath(workspaceDir: string, workspacePath: string): NormalizedWorkspaceAssetPath { + const safePath = assertSafeWorkspaceRelativePath(workspacePath) + const root = resolve(workspaceDir) + const absolutePath = resolve(root, ...safePath.split('/')) + const back = relative(root, absolutePath) + if (!back || back.startsWith('..') || isAbsolute(back)) throw new Error('Workspace library path escapes the workspace root') + return { workspacePath: safePath, absolutePath } +} + +function extensionOf(workspacePath: string): string { + return extname(workspacePath).slice(1).toLowerCase() +} + +function isGlbOrGltf(workspacePath: string): boolean { + return /\.(glb|gltf)$/i.test(workspacePath) +} + +function sourceScopeFor(workspacePath: string): AssetLibrarySourceScope { + return workspacePath.startsWith('Exports/') ? 'exports' : 'workflows' +} + +export function classifyAssetLibraryCandidate(candidate: AssetLibraryClassificationCandidate): AssetLibraryClassification { + const extension = extensionOf(candidate.workspacePath) + + if (candidate.workspacePath.endsWith('.landmarks.v1.json')) { + return { capability: 'landmarks-sidecar', state: 'ready', previewKind: 'text', openable: false, nonOpenableReason: 'Landmark sidecars require opening their source mesh.' } + } + if (candidate.workspacePath.endsWith('.world.json')) { + return { capability: 'generated-world', state: 'ready', previewKind: 'text', openable: false, nonOpenableReason: 'Generated worlds are list-only in this release.' } + } + if (candidate.workspacePath.endsWith('.scene.json')) { + return { capability: 'scene-manifest', state: 'ready', previewKind: 'text', openable: false, nonOpenableReason: 'Scene manifests are list-only in this release.' } + } + if (INTRINSIC_MOTION_EXTENSIONS.has(extension)) { + return { capability: 'animation-motion', state: 'ready', previewKind: 'binary', openable: false, nonOpenableReason: 'Motion files are list-only in this release.' } + } + if (extension === 'glb' || extension === 'gltf') { + return { capability: candidate.hasRigMetadata ? 'rigged-mesh' : 'mesh', state: 'ready', previewKind: '3d-model', openable: true } + } + if (MESH_EXTENSIONS.has(extension)) { + return { capability: 'mesh', state: 'ready', previewKind: 'binary', openable: false, nonOpenableReason: `.${extension} workspace assets are list-only in this release.` } + } + if (TEXT_EXTENSIONS.has(extension)) { + return { state: 'unsupported', previewKind: 'text', openable: false, nonOpenableReason: 'Unsupported workspace asset.' } + } + return { state: 'unsupported', previewKind: 'binary', openable: false, nonOpenableReason: 'Unsupported workspace asset.' } +} + +function stringField(value: unknown): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined +} + +function objectField(value: unknown): Record | undefined { + return typeof value === 'object' && value !== null && !Array.isArray(value) ? value as Record : undefined +} + +function manifestCapabilityFor(workspacePath: string): 'generated-world' | 'scene-manifest' | undefined { + if (workspacePath.endsWith('.world.json')) return 'generated-world' + if (workspacePath.endsWith('.scene.json')) return 'scene-manifest' + return undefined +} + +async function safeLinkedWorkspacePath(workspaceDir: string, ownerWorkspacePath: string, candidate: unknown, expected?: { mustBeMesh?: boolean, mustBeManifest?: boolean }): Promise<{ workspacePath?: string, warning?: string }> { + const raw = stringField(candidate) + if (!raw) return {} + let normalized: NormalizedWorkspaceAssetPath + try { + normalized = normalizeWorkspaceAssetPath(workspaceDir, raw) + } catch { + return { warning: 'Ignored unsafe linked workspace path.' } + } + if (normalized.workspacePath === ownerWorkspacePath) return { warning: 'Ignored self-linked workspace path.' } + if (expected?.mustBeMesh && !isGlbOrGltf(normalized.workspacePath)) return { warning: 'Ignored non-GLB/GLTF source workspace path.' } + if (expected?.mustBeManifest && !manifestCapabilityFor(normalized.workspacePath)) return { warning: 'Ignored non-manifest workspace path.' } + try { + await stat(normalized.absolutePath) + } catch { + return { warning: 'Ignored missing linked workspace path.' } + } + return { workspacePath: normalized.workspacePath } +} + +async function readMetadata(workspaceDir: string, workspacePath: string, absolutePath: string): Promise { + const metadata: AssetLibraryMetadata = { warnings: [] } + if (extensionOf(workspacePath) !== 'json') return metadata + + let parsed: Record + try { + parsed = JSON.parse(await readFile(absolutePath, 'utf8')) as Record + } catch { + return metadata + } + + const sourceRaw = parsed.sourceWorkspacePath ?? parsed.source_workspace_path ?? parsed.sourcePath ?? objectField(parsed.source)?.workspacePath + const source = await safeLinkedWorkspacePath(workspaceDir, workspacePath, sourceRaw, { mustBeMesh: true }) + if (source.workspacePath) metadata.sourceWorkspacePath = source.workspacePath + if (source.warning) metadata.warnings.push(source.warning) + + const manifestRaw = parsed.manifestWorkspacePath ?? parsed.manifest_workspace_path ?? objectField(parsed.manifest)?.workspacePath + const manifest = await safeLinkedWorkspacePath(workspaceDir, workspacePath, manifestRaw, { mustBeManifest: true }) + if (manifest.workspacePath) metadata.manifestWorkspacePath = manifest.workspacePath + if (manifest.warning) metadata.warnings.push(manifest.warning) + + metadata.artifactId = stringField(parsed.artifactId ?? parsed.artifact_id) + metadata.versionId = stringField(parsed.versionId ?? parsed.version_id) + metadata.provenance = objectField(parsed.provenance) as ArtifactProvenance | undefined + return metadata +} + +function shouldSkipDirectory(name: string): boolean { + return name.startsWith('.') || SKIPPED_DIRS.has(name.toLowerCase()) +} + +function shouldSkipFile(name: string): boolean { + return name.startsWith('.') || INTERNAL_SUFFIXES.some((suffix) => name.endsWith(suffix)) +} + +async function hasRigMetadata(absolutePath: string): Promise { + if (!/\.(glb|gltf)$/i.test(absolutePath)) return false + const stem = absolutePath.replace(/\.(glb|gltf)$/i, '') + try { + await stat(`${stem}.rigmeta.json`) + return true + } catch { + return false + } +} + +async function buildEntry(workspaceDir: string, workspacePath: string): Promise { + const { absolutePath } = normalizeWorkspaceAssetPath(workspaceDir, workspacePath) + const stats = await stat(absolutePath) + const classification = classifyAssetLibraryCandidate({ workspacePath, hasRigMetadata: await hasRigMetadata(absolutePath) }) + const metadata = await readMetadata(workspaceDir, workspacePath, absolutePath) + const manifestCapability = metadata.manifestWorkspacePath ? manifestCapabilityFor(metadata.manifestWorkspacePath) : undefined + return { + id: `library:${workspacePath}`, + workspacePath, + displayName: basename(workspacePath), + sourceScope: sourceScopeFor(workspacePath), + capability: classification.capability, + state: classification.state, + previewKind: classification.previewKind, + source: metadata.sourceWorkspacePath ? { workspacePath: metadata.sourceWorkspacePath, displayName: basename(metadata.sourceWorkspacePath), role: 'source-mesh' } : undefined, + manifest: metadata.manifestWorkspacePath && manifestCapability ? { workspacePath: metadata.manifestWorkspacePath, capability: manifestCapability } : undefined, + artifactId: metadata.artifactId, + versionId: metadata.versionId, + provenance: metadata.provenance, + warnings: metadata.warnings, + openable: classification.openable, + nonOpenableReason: classification.nonOpenableReason, + createdAt: (stats.birthtime.getTime() > 0 ? stats.birthtime : stats.mtime).toISOString(), + updatedAt: stats.mtime.toISOString(), + } +} + +async function collectFiles(workspaceDir: string, rootName: typeof ALLOWED_ROOTS[number]): Promise { + const root = join(workspaceDir, rootName) + const files: string[] = [] + + async function walk(dir: string): Promise { + let entries: Awaited> + try { + entries = await readdir(dir, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + if (entry.isDirectory()) { + if (!shouldSkipDirectory(entry.name)) await walk(join(dir, entry.name)) + continue + } + if (!entry.isFile() || shouldSkipFile(entry.name)) continue + files.push(relative(workspaceDir, join(dir, entry.name)).split(sep).join('/')) + } + } + + await walk(root) + return files +} + +export async function listWorkspaceAssetLibrary(request: WorkspaceAssetLibraryRequest): Promise { + try { + const workspacePaths = (await Promise.all(ALLOWED_ROOTS.map((root) => collectFiles(request.workspaceDir, root)))).flat().sort() + const entries = await Promise.all(workspacePaths.map((workspacePath) => buildEntry(request.workspaceDir, workspacePath))) + return { success: true, entries: entries.filter((entry) => entry.state !== 'unsupported') } + } catch (error) { + return { success: false, error: libraryError('list-failed', error instanceof Error ? error.message : String(error)) } + } +} + +async function previewEntry(absolutePath: string, entry: AssetLibraryEntry): Promise { + if (entry.previewKind === '3d-model') return { kind: '3d-model', viewerKind: extensionOf(entry.workspacePath) as 'glb' | 'gltf' } + const stats = await stat(absolutePath) + if (entry.previewKind === 'text') { + const maxBytes = 64 * 1024 + const content = await readFile(absolutePath, 'utf8') + return { kind: 'text', content: content.slice(0, maxBytes), byteLength: stats.size, truncated: content.length > maxBytes } + } + if (entry.previewKind === 'binary') { + return { kind: 'binary', binaryKind: extensionOf(entry.workspacePath), byteLength: stats.size, message: 'Binary preview is unavailable.' } + } + return { kind: 'none' } +} + +export async function readWorkspaceAssetLibraryEntry(request: WorkspaceAssetLibraryReadRequest): Promise { + try { + const normalized = normalizeWorkspaceAssetPath(request.workspaceDir, request.workspacePath) + const sourceValidation = await validateRequestedSource(request.workspaceDir, normalized.workspacePath, request.sourceWorkspacePath) + if (!sourceValidation.success) return { success: false, error: sourceValidation.error } + const entry = await buildEntry(request.workspaceDir, normalized.workspacePath) + return { success: true, entry, preview: await previewEntry(normalized.absolutePath, entry) } + } catch (error) { + return { success: false, error: libraryError('unsafe-path', error instanceof Error ? error.message : String(error)) } + } +} + +async function validateRequestedSource(workspaceDir: string, workspacePath: string, sourceWorkspacePath?: string): Promise<{ success: true } | { success: false, error: AssetLibraryError }> { + if (!sourceWorkspacePath) return { success: true } + let source: NormalizedWorkspaceAssetPath + try { + source = normalizeWorkspaceAssetPath(workspaceDir, sourceWorkspacePath) + } catch (error) { + return { success: false, error: libraryError('unsafe-path', error instanceof Error ? error.message : String(error)) } + } + if (source.workspacePath === workspacePath) return { success: false, error: libraryError('not-openable', 'Linked source must not point to the same workspace asset.') } + if (!isGlbOrGltf(source.workspacePath)) return { success: false, error: libraryError('not-openable', 'Linked source must be a safe .glb/.gltf workspace asset.') } + try { + await stat(source.absolutePath) + } catch { + return { success: false, error: libraryError('not-openable', 'Linked source workspace asset was not found.') } + } + const entry = await buildEntry(workspaceDir, workspacePath) + if (entry.source?.workspacePath !== source.workspacePath) { + return { success: false, error: libraryError('not-openable', 'Requested source does not match the indexed library source link.') } + } + return { success: true } +} + +export async function openWorkspaceAssetLibraryEntry(request: WorkspaceAssetLibraryReadRequest): Promise { + const read = await readWorkspaceAssetLibraryEntry(request) + if (!read.success) return read + if (request.sourceWorkspacePath) return { success: true, entry: read.entry } + if (!read.entry.openable) { + return { success: false, error: libraryError('not-openable', read.entry.nonOpenableReason ?? 'Workspace asset is not openable.') } + } + return { success: true, entry: read.entry } +} + +function readPayloadRequest(payload: unknown): { workspacePath?: string, sourceWorkspacePath?: string } { + if (typeof payload !== 'object' || payload === null) return {} + const values = payload as { workspacePath?: unknown, sourceWorkspacePath?: unknown } + return { + workspacePath: typeof values.workspacePath === 'string' ? values.workspacePath : undefined, + sourceWorkspacePath: typeof values.sourceWorkspacePath === 'string' ? values.sourceWorkspacePath : undefined, + } +} + +export function registerWorkspaceAssetLibraryIpcHandlers(deps: WorkspaceAssetLibraryIpcDeps): void { + deps.ipcMain.handle('workspace:library:list', async () => listWorkspaceAssetLibrary({ workspaceDir: deps.getWorkspaceDir() })) + deps.ipcMain.handle('workspace:library:read', async (_event, payload) => { + const { workspacePath, sourceWorkspacePath } = readPayloadRequest(payload) + if (!workspacePath) return { success: false, error: libraryError('invalid-request', 'workspacePath is required.') } + return readWorkspaceAssetLibraryEntry({ workspaceDir: deps.getWorkspaceDir(), workspacePath, sourceWorkspacePath }) + }) + deps.ipcMain.handle('workspace:library:open', async (_event, payload) => { + const { workspacePath, sourceWorkspacePath } = readPayloadRequest(payload) + if (!workspacePath) return { success: false, error: libraryError('invalid-request', 'workspacePath is required.') } + return openWorkspaceAssetLibraryEntry({ workspaceDir: deps.getWorkspaceDir(), workspacePath, sourceWorkspacePath }) + }) +} diff --git a/electron/main/extension-install-utils.test.mjs b/electron/main/extension-install-utils.test.mjs new file mode 100644 index 0000000..5a7e10d --- /dev/null +++ b/electron/main/extension-install-utils.test.mjs @@ -0,0 +1,62 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +function loadModule() { + const outfile = join(mkdtempSync(join(tmpdir(), 'modly-ext-test-')), 'extension-install-utils.cjs') + const require = createRequire(import.meta.url) + const result = buildSync({ + entryPoints: [resolve('electron/main/extension-install-utils.ts')], + bundle: true, + platform: 'node', + format: 'cjs', + write: false, + }) + writeFileSync(outfile, result.outputFiles[0].text, 'utf8') + return require(outfile) +} + +test('validateInstallManifest accepts legacy flat model manifests', () => { + const mod = loadModule() + + const validated = mod.validateInstallManifest( + { id: 'legacy-model', generator_class: 'Generator' }, + { + hasEntryFile: () => false, + hasGeneratorFile: () => true, + }, + 'repository', + ) + + assert.equal(validated.id, 'legacy-model') + assert.equal(validated.isProcess, false) + assert.equal(validated.hasNodes, false) +}) + +test('validateInstallManifest still rejects missing process entry files', () => { + const mod = loadModule() + + assert.throws( + () => mod.validateInstallManifest( + { id: 'proc', type: 'process', entry: 'processor.py' }, + { + hasEntryFile: () => false, + hasGeneratorFile: () => false, + }, + 'selected folder', + ), + /entry file "processor\.py" missing from selected folder/, + ) +}) + +test('python process setup failures are treated as fatal', () => { + const mod = loadModule() + + assert.equal(mod.isSetupFailureFatal({ isProcess: true, isPythonProcess: true }), true) + assert.equal(mod.isSetupFailureFatal({ isProcess: true, isPythonProcess: false }), false) + assert.equal(mod.isSetupFailureFatal({ isProcess: false, isPythonProcess: false }), true) +}) diff --git a/electron/main/extension-install-utils.ts b/electron/main/extension-install-utils.ts new file mode 100644 index 0000000..73e5cfa --- /dev/null +++ b/electron/main/extension-install-utils.ts @@ -0,0 +1,54 @@ +export interface InstallManifest { + id?: string + type?: 'model' | 'process' + entry?: string + generator_class?: string + nodes?: Array<{ id?: string }> +} + +export interface ValidatedInstallManifest { + id: string + isProcess: boolean + isPythonProcess: boolean + entryFile: string + hasNodes: boolean +} + +export function validateInstallManifest( + manifest: InstallManifest, + opts: { + hasEntryFile: (entryFile: string) => boolean + hasGeneratorFile: () => boolean + }, + sourceLabel: string, +): ValidatedInstallManifest { + if (!manifest.id) throw new Error('manifest.json: required field "id" missing') + + const isProcess = manifest.type === 'process' + const entryFile = manifest.entry ?? 'processor.js' + const nodes = Array.isArray(manifest.nodes) ? manifest.nodes.filter((node) => node?.id) : [] + + if (isProcess) { + if (!opts.hasEntryFile(entryFile)) { + throw new Error(`manifest.json: entry file "${entryFile}" missing from ${sourceLabel}`) + } + } else { + if (!opts.hasGeneratorFile()) throw new Error(`generator.py missing from ${sourceLabel}`) + if (!manifest.generator_class) throw new Error('manifest.json: required field "generator_class" missing') + } + + return { + id: manifest.id, + isProcess, + isPythonProcess: isProcess && entryFile.endsWith('.py'), + entryFile, + hasNodes: nodes.length > 0, + } +} + +export function isSetupFailureFatal(kind: { + isProcess: boolean + isPythonProcess: boolean +}): boolean { + return !kind.isProcess || kind.isPythonProcess +} diff --git a/electron/main/index.ts b/electron/main/index.ts index 19b75df..6ef0b46 100644 --- a/electron/main/index.ts +++ b/electron/main/index.ts @@ -33,6 +33,21 @@ function createWindow(): void { mainWindow?.show() }) + mainWindow.webContents.on('before-input-event', (event, input) => { + const isMacQuitShortcut = + process.platform === 'darwin' && + input.type === 'keyDown' && + input.key.toLowerCase() === 'q' && + input.meta && + !input.control && + !input.alt + + if (isMacQuitShortcut) { + event.preventDefault() + app.quit() + } + }) + mainWindow.webContents.setWindowOpenHandler((details) => { shell.openExternal(details.url) return { action: 'deny' } @@ -88,7 +103,10 @@ app.whenReady().then(async () => { }) app.on('window-all-closed', () => { - if (process.platform !== 'darwin') app.quit() + // Modly holds a multi-GB Python subprocess; leaving it running in the + // Dock after the window closes (the Mac default) is the wrong behavior + // for this app. Closing the window means quit. + app.quit() }) app.on('before-quit', (event) => { diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 757b566..88fe240 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -2,10 +2,12 @@ import { ipcMain, BrowserWindow, dialog, app, shell } from 'electron' import { buildSync } from 'esbuild' import { autoUpdater } from 'electron-updater' import { join } from 'path' -import { rm as rmAsync, readFile, writeFile, mkdir, readdir, rename, cp } from 'fs/promises' +import { rm as rmAsync, readFile, writeFile, mkdir, readdir, rename, cp, symlink, lstat } from 'fs/promises' import { existsSync, readdirSync, statSync } from 'fs' import axios from 'axios' import * as tar from 'tar' +import * as os from 'os' +import { promisify } from 'util' import { PythonBridge, API_BASE_URL } from './python-bridge' import { isModelDownloaded, @@ -17,16 +19,27 @@ import { checkSetupNeeded, markSetupDone, runFullSetup, getVenvPythonExe, ensure import { logger } from './logger' import { getProcessRunner, getPythonProcessRunner, getExtPythonExe, terminateProcessRunner, terminateAllProcessRunners } from './process-runner' import { getBuiltinExtensionsDir } from './builtin-sync' -import { spawn } from 'child_process' +import { spawn, execFile } from 'child_process' import { assertSafeExtensionId, buildExtensionBackupPath, resolveExtensionPathWithinRoot } from './extension-path-guard' +import { isSetupFailureFatal, validateInstallManifest } from './extension-install-utils' +import { registerWorkspaceAssetLibraryIpcHandlers } from './artifact-registry-service' type WindowGetter = () => BrowserWindow | null +const pExecFile = promisify(execFile) // ─── GPU detect (best-effort, no Python required) ───────────────────────────── -interface GpuInfo { sm: number; cudaVersion: number } +interface GpuInfo { + sm: number + cudaVersion: number + accelerator: 'cuda' | 'mps' | 'cpu' +} function detectGpuInfo(): Promise { + if (process.platform === 'darwin' && process.arch === 'arm64') { + return Promise.resolve({ sm: 0, cudaVersion: 0, accelerator: 'mps' }) + } + return new Promise((resolve) => { // Query compute cap + driver version in one call const proc = spawn('nvidia-smi', ['--query-gpu=compute_cap,driver_version', '--format=csv,noheader'], { @@ -53,12 +66,12 @@ function detectGpuInfo(): Promise { else if (driverMajor >= 530) cudaVersion = 121 else if (driverMajor >= 525) cudaVersion = 120 else if (driverMajor >= 520) cudaVersion = 118 - resolve({ sm: isNaN(sm) ? 86 : sm, cudaVersion }) + resolve({ sm: isNaN(sm) ? 86 : sm, cudaVersion, accelerator: 'cuda' }) } else { - resolve({ sm: 86, cudaVersion: 118 }) + resolve({ sm: 0, cudaVersion: 0, accelerator: 'cpu' }) } }) - proc.on('error', () => resolve({ sm: 86, cudaVersion: 118 })) + proc.on('error', () => resolve({ sm: 0, cudaVersion: 0, accelerator: 'cpu' })) }) } @@ -76,8 +89,92 @@ function runExtensionSetup( const pythonExe = getVenvPythonExe(userData) const setupPy = join(extDir, 'setup.py') - const args = JSON.stringify({ python_exe: pythonExe, ext_dir: extDir, gpu_sm: gpuSm, cuda_version: cudaVersion }) - const proc = spawn(pythonExe, [setupPy, args], { + const accelerator = process.platform === 'darwin' && process.arch === 'arm64' ? 'mps' : gpuSm > 0 ? 'cuda' : 'cpu' + const args = JSON.stringify({ + python_exe: pythonExe, + ext_dir: extDir, + gpu_sm: gpuSm, + cuda_version: cudaVersion, + accelerator, + platform: process.platform, + arch: process.arch, + }) + const launcher = ` +import runpy +import subprocess +import sys + +setup_py = sys.argv[1] +setup_args = sys.argv[2:] + +_original_run = subprocess.run +_original_check_call = subprocess.check_call +_original_check_output = subprocess.check_output + +def _is_cuda_torch_index(value): + return isinstance(value, str) and value.startswith("https://download.pytorch.org/whl/cu") + +def _mentions_torch(command): + if not isinstance(command, (list, tuple)): + return False + return any(str(part).startswith(("torch==", "torchvision==", "torchaudio==")) for part in command) + +def _rewrite_command(command): + if sys.platform != "darwin" or not _mentions_torch(command): + return command + if not isinstance(command, (list, tuple)): + return command + + rewritten = [] + changed = False + i = 0 + while i < len(command): + part = command[i] + text = str(part) + if text in ("--index-url", "-i", "--extra-index-url") and i + 1 < len(command) and _is_cuda_torch_index(str(command[i + 1])): + changed = True + i += 2 + continue + if text.startswith("--index-url=") or text.startswith("--extra-index-url="): + value = text.split("=", 1)[1] + if _is_cuda_torch_index(value): + changed = True + i += 1 + continue + rewritten.append(part) + i += 1 + + if changed: + print("[Modly setup compat] Removed CUDA-only PyTorch index on macOS; pip will use macOS wheels.", file=sys.stderr) + return rewritten + return command + +def _patched_run(*args, **kwargs): + args = list(args) + if args: + args[0] = _rewrite_command(args[0]) + return _original_run(*args, **kwargs) + +def _patched_check_call(*args, **kwargs): + args = list(args) + if args: + args[0] = _rewrite_command(args[0]) + return _original_check_call(*args, **kwargs) + +def _patched_check_output(*args, **kwargs): + args = list(args) + if args: + args[0] = _rewrite_command(args[0]) + return _original_check_output(*args, **kwargs) + +subprocess.run = _patched_run +subprocess.check_call = _patched_check_call +subprocess.check_output = _patched_check_output + +sys.argv = [setup_py] + setup_args +runpy.run_path(setup_py, run_name="__main__") +` + const proc = spawn(pythonExe, ['-c', launcher, setupPy, args], { stdio: ['ignore', 'pipe', 'pipe'], }) @@ -145,7 +242,12 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe ipcMain.handle('setup:check', async () => { const userData = app.getPath('userData') const defaultDataDir = join(app.getPath('documents'), 'Modly') - return { needed: checkSetupNeeded(userData), defaultDataDir } + return { + needed: checkSetupNeeded(userData), + defaultDataDir, + platform: process.platform, + arch: process.arch, + } }) ipcMain.handle('setup:saveDataDir', async (_event, { baseDir }: { baseDir: string }) => { @@ -207,7 +309,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe const result = await dialog.showOpenDialog(win, { title: 'Select a 3D mesh file', - filters: [{ name: '3D Mesh', extensions: ['glb', 'obj', 'stl', 'ply'] }], + filters: [{ name: '3D Mesh', extensions: ['glb', 'obj', 'stl', 'ply', 'splat'] }], properties: ['openFile'] }) @@ -253,17 +355,39 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe ipcMain.handle('model:delete', async (_, modelId: string): Promise<{ success: boolean; error?: string }> => { const modelDir = join(getSettings(app.getPath('userData')).modelsDir, modelId) + + // Unload the model and wait for confirmation so file handles are released try { - await axios.post(`${API_BASE_URL}/model/unload/${encodeURIComponent(modelId)}`, {}, { timeout: 5000 }) + await axios.post(`${API_BASE_URL}/model/unload/${encodeURIComponent(modelId)}`, {}, { timeout: 10_000 }) + // Give the OS a moment to release file locks (Windows holds handles briefly after close) + await new Promise(resolve => setTimeout(resolve, 1_500)) } catch { - // unload is best-effort β€” proceed with deletion anyway + // Unload failed (model may not be loaded) β€” still attempt deletion } - try { - await rmAsync(modelDir, { recursive: true, force: true }) - return { success: true } - } catch (err) { - return { success: false, error: String(err) } + + // Retry removal β€” Windows may return EBUSY/EPERM if handles linger + const maxRetries = 3 + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await rmAsync(modelDir, { recursive: true, force: true }) + return { success: true } + } catch (err: unknown) { + const code = (err as NodeJS.ErrnoException).code + const isLocked = code === 'EBUSY' || code === 'EPERM' + if (isLocked && attempt < maxRetries) { + await new Promise(resolve => setTimeout(resolve, 1_000 * attempt)) + continue + } + return { + success: false, + error: isLocked + ? `Model files are still locked after ${maxRetries} attempts. Close any programs using the model and try again.` + : String(err), + } + } } + + return { success: false, error: 'Unexpected error during deletion' } }) ipcMain.handle('model:showInFolder', (_, modelId: string) => { @@ -275,6 +399,9 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe // Read local file β†’ base64 (bypasses file:// restrictions in the renderer) ipcMain.handle('fs:readFileBase64', async (_, filePath: string) => { + if (typeof filePath !== 'string' || filePath.trim().length === 0) { + throw new Error('fs:readFileBase64 requires a non-empty file path') + } const buffer = await readFile(filePath) return buffer.toString('base64') }) @@ -293,22 +420,65 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe return listDownloadedModels(modelsDir) }) - ipcMain.handle('model:isDownloaded', (_, modelId: string): boolean => { + ipcMain.handle('model:isDownloaded', (_, modelId: string, downloadCheck?: string): boolean => { const modelsDir = getSettings(app.getPath('userData')).modelsDir - return isModelDownloaded(modelsDir, modelId) + return isModelDownloaded(modelsDir, modelId, downloadCheck) }) ipcMain.handle('model:activeDownloads', () => [...activeDownloads.entries()].map(([modelId, progress]) => ({ modelId, ...progress })) ) - ipcMain.handle('model:download', async (event, { repoId, modelId, skipPrefixes }: { repoId: string; modelId: string; skipPrefixes?: string[] }) => { + ipcMain.handle('model:download', async ( + event, + { repoId, modelId, skipPrefixes, includePrefixes }: { repoId: string; modelId: string; skipPrefixes?: string[]; includePrefixes?: string[] }, + ) => { + if (activeDownloads.has(modelId)) { + return { success: false, error: 'Download already in progress' } + } activeDownloads.set(modelId, { percent: 0 }) try { await downloadModelFromHF(repoId, modelId, (progress) => { activeDownloads.set(modelId, progress) event.sender.send('model:downloadProgress', { modelId, ...progress }) - }, skipPrefixes) + }, skipPrefixes, includePrefixes) + return { success: true } + } catch (err: any) { + const message = err?.message ?? String(err) + if (message.includes('paused')) { + event.sender.send('model:downloadProgress', { modelId, percent: 0, status: 'paused', paused: true }) + return { success: false, paused: true } + } + if (message.includes('cancelled')) { + event.sender.send('model:downloadProgress', { modelId, percent: 0, status: 'cancelled', cancelled: true }) + return { success: false, cancelled: true } + } + return { success: false, error: String(err) } + } finally { + activeDownloads.delete(modelId) + } + }) + + ipcMain.handle('model:pauseDownload', async (_, modelId: string): Promise<{ success: boolean; error?: string }> => { + try { + await axios.post(`${API_BASE_URL}/model/hf-download/pause`, null, { + params: { model_id: modelId }, + timeout: 5000, + }) + return { success: true } + } catch (err) { + return { success: false, error: String(err) } + } + }) + + ipcMain.handle('model:cancelDownload', async (_, modelId: string): Promise<{ success: boolean; error?: string }> => { + try { + await axios.post(`${API_BASE_URL}/model/hf-download/cancel`, null, { + params: { model_id: modelId }, + timeout: 5000, + }) + const modelDir = join(getSettings(app.getPath('userData')).modelsDir, modelId) + await rmAsync(modelDir, { recursive: true, force: true }) return { success: true } } catch (err) { return { success: false, error: String(err) } @@ -348,11 +518,46 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe ipcMain.handle('shell:openExternal', (_, url: string) => shell.openExternal(url)) // App info + // System memory (used/available/total bytes). + // On macOS, matches Activity Monitor's "Memory Used": + // used = wired + active + compressed. + ipcMain.handle('system:memory', async () => { + const total = os.totalmem() + + if (process.platform === 'darwin') { + try { + const { stdout } = await pExecFile('vm_stat', []) + const pageSizeMatch = stdout.match(/page size of (\d+) bytes/) + const pageSize = pageSizeMatch ? parseInt(pageSizeMatch[1]!, 10) : 16384 + + const pagesFor = (label: string): number => { + const m = stdout.match(new RegExp(`${label}:\\s+(\\d+)`)) + return m ? parseInt(m[1]!, 10) : 0 + } + + const active = pagesFor('Pages active') + const wired = pagesFor('Pages wired down') + const compressed = pagesFor('Pages occupied by compressor') + + const used = (active + wired + compressed) * pageSize + const available = Math.max(0, total - used) + return { total, used, available } + } catch { + // Fall back to total - free outside Activity Monitor semantics. + } + } + + const free = os.freemem() + return { total, used: total - free, available: free } + }) + ipcMain.handle('app:info', () => ({ version: app.getVersion(), userData: app.getPath('userData'), modelsDir: getSettings(app.getPath('userData')).modelsDir, - apiUrl: API_BASE_URL + apiUrl: API_BASE_URL, + platform: process.platform, + arch: process.arch, })) // Settings β€” seed HF token into main-process env at startup @@ -385,11 +590,12 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe }) // Directory picker - ipcMain.handle('fs:selectDirectory', async () => { + ipcMain.handle('fs:selectDirectory', async (_event, defaultPath?: string) => { const win = getWindow() if (!win) return null const result = await dialog.showOpenDialog(win, { properties: ['openDirectory', 'createDirectory'], + ...(defaultPath && { defaultPath }), }) return result.canceled ? null : result.filePaths[0] }) @@ -415,6 +621,11 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe const workspacePath = (...parts: string[]) => join(getSettings(app.getPath('userData')).workspaceDir, ...parts) + registerWorkspaceAssetLibraryIpcHandlers({ + ipcMain, + getWorkspaceDir: () => getSettings(app.getPath('userData')).workspaceDir, + }) + ipcMain.handle('workspace:listCollections', async () => { const base = workspacePath() await mkdir(base, { recursive: true }) @@ -499,7 +710,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe }) // Remote registry β€” list of trusted GitHub repo URLs - const REGISTRY_URL = 'https://raw.githubusercontent.com/liightnig125/modly-official-extension/main/registry.json' + const REGISTRY_URL = 'https://raw.githubusercontent.com/lightningpixel/modly-official-extension/main/registry.json' const REGISTRY_TTL = 5 * 60 * 1000 // 5 minutes let registryCache: { repos: Set; fetchedAt: number } | null = null @@ -537,6 +748,9 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe // extension type type?: 'model' | 'process' entry?: string + // Optional top-level fallbacks β€” applied to each node if not set on the node + params_schema?: unknown[] + param_defaults?: Record nodes?: { id: string name?: string @@ -544,9 +758,11 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe inputs?: ('mesh' | 'image' | 'text')[] output?: 'mesh' | 'image' | 'text' params_schema?: unknown[] + param_defaults?: Record hf_repo?: string download_check?: string hf_skip_prefixes?: string[] + hf_include_prefixes?: string[] }[] } @@ -568,10 +784,12 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe input: n.input ?? 'image' as const, inputs: n.inputs, output: n.output ?? 'mesh' as const, - paramsSchema: n.params_schema ?? [], + paramsSchema: n.params_schema ?? parsed.params_schema ?? [], + paramDefaults: { ...(parsed.param_defaults ?? {}), ...(n.param_defaults ?? {}) }, hfRepo: n.hf_repo, downloadCheck: n.download_check, hfSkipPrefixes: n.hf_skip_prefixes, + hfIncludePrefixes: n.hf_include_prefixes, })) if (parsed.type === 'process') { @@ -593,15 +811,38 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe if (!existsSync(dir)) return [] try { const entries = await readdir(dir, { withFileTypes: true }) - const dirs = entries.filter(e => e.isDirectory()) + // On Windows, junction points are reported by Node.js as isSymbolicLink()=true, + // isDirectory()=false. Use statSync (which follows links) as the authoritative check. + const dirs = entries.filter(e => { + if (e.isDirectory()) return true + if (e.isSymbolicLink()) { + try { return statSync(join(dir, e.name)).isDirectory() } catch { return false } + } + return false + }) return Promise.all(dirs.map(async (entry) => { const base = { type: 'model' as const, id: entry.name, name: entry.name, trusted: isBuiltin, builtin: isBuiltin, nodes: [] } + const entryPath = join(dir, entry.name) + + // Detect local extensions: check for .modly-local sentinel + let localSourcePath: string | undefined + if (!isBuiltin) { + const sentinelPath = join(entryPath, '.modly-local') + if (existsSync(sentinelPath)) { + try { + localSourcePath = (await readFile(sentinelPath, 'utf-8')).trim() + } catch { /* ignore */ } + } + } + for (const manifestFile of ['manifest.json', 'package.json']) { - const p = join(dir, entry.name, manifestFile) + const p = join(entryPath, manifestFile) if (existsSync(p)) { try { const raw = await readFile(p, 'utf-8') const parsed = JSON.parse(raw) as ParsedManifest + // Inject local:// source so the UI shows the Local badge + if (localSourcePath) parsed.source = `local://${localSourcePath}` return parseExtensionManifest(parsed, entry.name, trustedRepos, isBuiltin) } catch { /* ignore parse errors, fall through */ } } @@ -674,25 +915,17 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe const manifestRaw = await readFile(manifestPath, 'utf-8') const manifest = JSON.parse(manifestRaw) as ParsedManifest - if (!manifest.id) throw new Error('manifest.json: required field "id" missing') - const extensionId = assertSafeExtensionId(manifest.id) + const { id: rawManifestId, isProcess, entryFile, isPythonProcess, hasNodes } = validateInstallManifest( + manifest, + { + hasEntryFile: (candidate) => existsSync(join(extractDir, candidate)), + hasGeneratorFile: () => existsSync(join(extractDir, 'generator.py')), + }, + 'repository', + ) + if (!hasNodes) throw new Error('manifest.json: required field "nodes" missing or empty') + const extensionId = assertSafeExtensionId(rawManifestId) manifest.id = extensionId - if (!manifest.nodes?.length) throw new Error('manifest.json: required field "nodes" missing or empty') - - const isProcess = manifest.type === 'process' - const entryFile = manifest.entry ?? 'processor.js' - const isPythonProcess = isProcess && entryFile.endsWith('.py') - - if (isProcess) { - // Process extension validation - if (!existsSync(join(extractDir, entryFile))) - throw new Error(`manifest.json: entry file "${entryFile}" missing from repository`) - } else { - // Model extension validation - const generatorPath = join(extractDir, 'generator.py') - if (!existsSync(generatorPath)) throw new Error('generator.py missing from repository') - if (!manifest.generator_class) throw new Error('manifest.json: required field "generator_class" missing') - } // Override source field with the actual GitHub URL so trust is based on origin manifest.source = `https://github.com/${owner}/${repo}` @@ -732,10 +965,18 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe if (existsSync(join(destDir, 'setup.py'))) { emit({ step: 'setting_up', message: 'Setting up Python environment…' }) const { sm: gpuSm, cudaVersion } = await detectGpuInfo() - await runExtensionSetup(destDir, gpuSm, cudaVersion, (line) => { - logger.info(`[ext-setup] ${line}`) - emit({ step: 'setting_up', message: line }) - }) + try { + await runExtensionSetup(destDir, gpuSm, cudaVersion, (line) => { + logger.info(`[ext-setup] ${line}`) + emit({ step: 'setting_up', message: line }) + }) + } catch (err) { + if (isSetupFailureFatal({ isProcess, isPythonProcess })) { + throw new Error(`Extension setup failed: ${err}`) + } + logger.warn(`[ext-setup] setup.py failed: ${err}`) + emit({ step: 'setting_up', message: `Warning: setup failed β€” ${err}` }) + } } } else if (isProcess) { // 6b. JS process extension: npm install if package.json present @@ -851,6 +1092,99 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe } }) + // Install a local extension by creating a symlink/junction to a local folder + ipcMain.handle('extensions:installFromLocal', async (event) => { + const win = getWindow() + const emit = (data: object) => win?.webContents.send('extensions:installProgress', data) + + // 1. Open folder picker + const pickResult = await dialog.showOpenDialog(win!, { + title: 'Select local extension folder', + properties: ['openDirectory'], + }) + if (pickResult.canceled || pickResult.filePaths.length === 0) { + return { success: false, cancelled: true } + } + const localPath = pickResult.filePaths[0] + + try { + emit({ step: 'validating' }) + + // 2. Read and validate manifest.json + const manifestPath = join(localPath, 'manifest.json') + if (!existsSync(manifestPath)) { + throw new Error('manifest.json not found in the selected folder') + } + const manifestRaw = await readFile(manifestPath, 'utf-8') + const manifest = JSON.parse(manifestRaw) as ParsedManifest + + const { id: rawManifestId } = validateInstallManifest( + manifest, + { + hasEntryFile: (candidate) => existsSync(join(localPath, candidate)), + hasGeneratorFile: () => existsSync(join(localPath, 'generator.py')), + }, + 'local folder', + ) + const extensionId = assertSafeExtensionId(rawManifestId) + + // 3. Create symlink / junction in extensionsDir + const userData = app.getPath('userData') + const extensionsDir = getSettings(userData).extensionsDir + await mkdir(extensionsDir, { recursive: true }) + + const linkPath = resolveExtensionPathWithinRoot(extensionsDir, extensionId) + + // Remove any existing symlink/dir at that location + if (existsSync(linkPath)) { + // Check if it's already linked to the same path + try { + const stat = await lstat(linkPath) + if (stat.isSymbolicLink() || (process.platform === 'win32' && stat.isDirectory())) { + await rmAsync(linkPath, { recursive: true, force: true }) + } else { + throw new Error(`A non-symlink folder named "${extensionId}" already exists in extensionsDir. Remove it first.`) + } + } catch (e: any) { + if (e.message?.includes('already exists')) throw e + await rmAsync(linkPath, { recursive: true, force: true }) + } + } + + emit({ step: 'setting_up', message: 'Linking local folder…' }) + + if (process.platform === 'win32') { + // Windows: junction (no elevation needed, works for directories) + await symlink(localPath, linkPath, 'junction') + } else { + // macOS / Linux: standard symlink + await symlink(localPath, linkPath) + } + + // Write sentinel so extensions:list can detect this as a local extension + // The sentinel lives in the linked folder (= the original local folder), so + // it persists even if Modly is restarted. The content is the absolute path. + await writeFile(join(linkPath, '.modly-local'), localPath, 'utf-8') + + // 4. Hot-reload Python registry so it picks up the new extension + try { + await axios.post(`${API_BASE_URL}/extensions/reload`, {}, { timeout: 10_000 }) + } catch { /* Python might not be running yet */ } + + emit({ step: 'done', extensionId }) + + const trustedRepos = await fetchTrustedRepos() + // Build manifest with localPath marker so the UI can identify local extensions + const annotatedManifest = { ...manifest, source: `local://${localPath}` } + const ext = parseExtensionManifest(annotatedManifest, extensionId, trustedRepos) + return { success: true, extensionId, extension: ext, localPath } + + } catch (err) { + emit({ step: 'error', message: String(err) }) + return { success: false, error: String(err) } + } + }) + // Trigger Python extension reload (without touching the filesystem) ipcMain.handle('extensions:reload', async () => { terminateAllProcessRunners() diff --git a/electron/main/model-downloader.ts b/electron/main/model-downloader.ts index 4d6f76c..8c5571d 100644 --- a/electron/main/model-downloader.ts +++ b/electron/main/model-downloader.ts @@ -13,6 +13,9 @@ export interface DownloadProgress { fileIndex?: number totalFiles?: number status?: string + bytesDownloaded?: number + totalBytes?: number + stalledSeconds?: number } export type ProgressCallback = (progress: DownloadProgress) => void @@ -25,9 +28,12 @@ const PYTHON_API_URL = process.env['PYTHON_API_URL'] ?? 'http://127.0.0.1:8765' /** * Check if a model is already downloaded (directory exists and is non-empty). */ -export function isModelDownloaded(modelsDir: string, modelId: string): boolean { +export function isModelDownloaded(modelsDir: string, modelId: string, downloadCheck?: string): boolean { const modelDir = join(modelsDir, modelId) if (!existsSync(modelDir)) return false + if (downloadCheck && downloadCheck.trim()) { + return existsSync(join(modelDir, downloadCheck)) + } try { return readdirSync(modelDir).length > 0 } catch { @@ -112,12 +118,17 @@ export async function downloadModelFromHF( modelId: string, onProgress: ProgressCallback, skipPrefixes?: string[], + includePrefixes?: string[], ): Promise { const { net } = require('electron') + const STALL_TIMEOUT_MS = 120_000 let url = `${PYTHON_API_URL}/model/hf-download?repo_id=${encodeURIComponent(repoId)}&model_id=${encodeURIComponent(modelId)}` if (skipPrefixes && skipPrefixes.length > 0) { url += `&skip_prefixes=${encodeURIComponent(JSON.stringify(skipPrefixes))}` } + if (includePrefixes && includePrefixes.length > 0) { + url += `&include_prefixes=${encodeURIComponent(JSON.stringify(includePrefixes))}` + } const hfToken = getSettings(app.getPath('userData')).hfToken if (hfToken) { url += `&token=${encodeURIComponent(hfToken)}` @@ -131,8 +142,17 @@ export async function downloadModelFromHF( const reader = res.body.getReader() let buffer = '' + async function readWithTimeout() { + return await Promise.race([ + reader.read(), + new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Model download stalled for ${Math.round(STALL_TIMEOUT_MS / 1000)}s`)), STALL_TIMEOUT_MS) + }), + ]) + } + while (true) { - const { done, value } = await reader.read() + const { done, value } = await readWithTimeout() if (done) break buffer += decoder.decode(value, { stream: true }) @@ -149,10 +169,22 @@ export async function downloadModelFromHF( fileIndex: data.fileIndex, totalFiles: data.totalFiles, status: data.status, + bytesDownloaded: data.bytesDownloaded, + totalBytes: data.totalBytes, + stalledSeconds: data.stalledSeconds, }) + if (data.paused) throw new Error('Model download paused') + if (data.cancelled) throw new Error('Model download cancelled') if (data.error) throw new Error(`HF download error: ${data.error}`) } catch (e) { - if (e instanceof Error && e.message.startsWith('HF download error:')) throw e + if ( + e instanceof Error && + ( + e.message.startsWith('HF download error:') || + e.message === 'Model download paused' || + e.message === 'Model download cancelled' + ) + ) throw e } } } diff --git a/electron/main/python-bridge.ts b/electron/main/python-bridge.ts index f0451ce..94dcfb6 100644 --- a/electron/main/python-bridge.ts +++ b/electron/main/python-bridge.ts @@ -51,15 +51,21 @@ export class PythonBridge { cwd: apiDir, env: { ...cleanPythonEnv(), - PYTHONUNBUFFERED: '1', - // No PYTHONPATH needed β€” the venv's Python has its own isolated site-packages - MODELS_DIR: this.resolveModelsDir(), - WORKSPACE_DIR: this.resolveWorkspaceDir(), - EXTENSIONS_DIR: this.resolveExtensionsDir(), - ...(process.env['SELECTED_MODEL_ID'] ? { SELECTED_MODEL_ID: process.env['SELECTED_MODEL_ID'] } : {}), - HUGGING_FACE_HUB_TOKEN: this.resolveHfToken(), - HF_TOKEN: this.resolveHfToken(), - } + PYTHONUNBUFFERED: '1', + // No PYTHONPATH needed - the venv's Python has its own isolated site-packages + MODELS_DIR: this.resolveModelsDir(), + WORKSPACE_DIR: this.resolveWorkspaceDir(), + EXTENSIONS_DIR: this.resolveExtensionsDir(), + SELECTED_MODEL_ID: process.env['SELECTED_MODEL_ID'] ?? '', + HUGGING_FACE_HUB_TOKEN: this.resolveHfToken(), + HF_TOKEN: this.resolveHfToken(), + }, + // On Unix, put the bridge in its own process group so every subprocess + // it spawns (extension runners, etc.) inherits that group. On shutdown + // we SIGKILL the whole group (negative PID) to take them all out + // together β€” otherwise children get reparented to launchd and keep + // holding MPS-wired memory until the user kills them manually. + detached: process.platform !== 'win32', }) this.process.stdout?.on('data', (data) => { @@ -82,7 +88,13 @@ export class PythonBridge { this.ready = false this.process = null if (wasReady && !this.intentionalStop) { - this.getWindow()?.webContents.send('python:crashed', { code }) + const getWindow = this.getWindow + if (!getWindow) return + const win = getWindow() + const contents = win?.webContents + if (contents && !contents.isDestroyed()) { + contents.send('python:crashed', { code }) + } } }) @@ -97,8 +109,17 @@ export class PythonBridge { if (process.platform === 'win32') { const { execSync } = require('child_process') try { execSync(`taskkill /PID ${proc.pid} /T /F`) } catch {} - } else { - proc.kill('SIGTERM') + } else if (proc.pid) { + // Kill the entire process group (negative PID) so extension subprocesses + // die with the bridge instead of being orphaned to launchd. SIGKILL + // rather than SIGTERM: on app quit we want immediate release of Metal + // wired memory, not a polite request the subprocess might ignore while + // it finishes an operation. + try { + process.kill(-proc.pid, 'SIGKILL') + } catch { + try { proc.kill('SIGKILL') } catch {} + } } console.log('[PythonBridge] Stopped') } @@ -114,7 +135,13 @@ export class PythonBridge { private emitTqdmLog(raw: string): void { if (/INFO/.test(raw)) return if (!raw.trim()) return - this.getWindow()?.webContents.send('python:log', raw.trim()) + const getWindow = this.getWindow + if (!getWindow) return + const win = getWindow() + const contents = win?.webContents + if (contents && !contents.isDestroyed()) { + contents.send('python:log', raw.trim()) + } } isReady(): boolean { return this.ready } diff --git a/electron/main/python-setup.ts b/electron/main/python-setup.ts index 5604487..3b85db1 100644 --- a/electron/main/python-setup.ts +++ b/electron/main/python-setup.ts @@ -1,5 +1,6 @@ import { BrowserWindow, app } from 'electron' import { existsSync, readFileSync, writeFileSync } from 'fs' +import { cp, rm, mkdir } from 'fs/promises' import { join } from 'path' import { spawn, execSync } from 'child_process' import { createHash } from 'crypto' @@ -150,13 +151,46 @@ export function markSetupDone(userData: string): void { ) } +// ─── AppImage: stable Python runtime ───────────────────────────────────────── + +/** + * On Linux AppImage, process.resourcesPath resolves to an ephemeral mount point + * (/tmp/.mount_Modly-XXXXXX/) that changes every launch, which breaks venv symlinks. + * This copies the bundled Python runtime to a stable userData path once per app version. + */ +async function ensureStableEmbeddedPython(userData: string, win: BrowserWindow): Promise { + const stableDir = join(userData, 'python-embed') + const stableExe = join(stableDir, 'bin', 'python3') + const versionFile = join(stableDir, '.app-version') + const currentVersion = app.getVersion() + + const alreadyCopied = + existsSync(stableExe) && + existsSync(versionFile) && + readFileSync(versionFile, 'utf-8').trim() === currentVersion + + if (!alreadyCopied) { + win.webContents.send('setup:progress', { step: 'venv', percent: 2 }) + console.log('[PythonSetup] Extracting Python runtime to stable path:', stableDir) + if (existsSync(stableDir)) { + await rm(stableDir, { recursive: true, force: true }) + } + await mkdir(stableDir, { recursive: true }) + await cp(getEmbeddedPythonDir(), stableDir, { recursive: true, preserveTimestamps: true }) + writeFileSync(versionFile, currentVersion, 'utf-8') + console.log('[PythonSetup] Python runtime ready at:', stableDir) + } + + return stableExe +} + // ─── Setup steps ───────────────────────────────────────────────────────────── function createVenv(pythonExe: string, venvDir: string, win: BrowserWindow): Promise { return new Promise((resolve, reject) => { win.webContents.send('setup:progress', { step: 'venv', percent: 5 }) console.log('[PythonSetup] Creating venv at', venvDir) - const proc = spawn(pythonExe, ['-m', 'venv', venvDir], { + const proc = spawn(pythonExe, ['-m', 'venv', '--clear', venvDir], { stdio: ['ignore', 'pipe', 'pipe'], env: cleanPythonEnv(), }) @@ -263,10 +297,14 @@ export async function runFullSetup(win: BrowserWindow, userData: string): Promis const requirementsPath = getRequirementsPath() const venvDir = getVenvDir(userData) - if (process.platform === 'win32' || app.isPackaged) { - // Packaged (all platforms) + Windows dev: use bundled python-build-standalone. - // python-build-standalone is a full Python install β†’ venv module works natively, - // DLLs come from the installer so SAC doesn't block them. + if (process.platform === 'linux' && app.isPackaged) { + // AppImage: process.resourcesPath is ephemeral β€” copy Python to stable userData path first + const pythonExe = await ensureStableEmbeddedPython(userData, win) + await createVenv(pythonExe, venvDir, win) + const venvPython = getVenvPythonExe(userData) + await installRequirements(venvPython, requirementsPath, win) + } else if (process.platform === 'win32' || app.isPackaged) { + // Windows (dev + packaged) or macOS packaged: bundled Python path is stable const pythonExe = getEmbeddedPythonExe() if (!existsSync(pythonExe)) { throw new Error( diff --git a/electron/preload/artifact-registry-preload.test.ts b/electron/preload/artifact-registry-preload.test.ts new file mode 100644 index 0000000..511589e --- /dev/null +++ b/electron/preload/artifact-registry-preload.test.ts @@ -0,0 +1,45 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { createElectronApi } from './electron-api.ts' + +test('preload exposes scoped workspace library list/read/open methods', async () => { + const calls: Array<{ channel: string, payload?: unknown }> = [] + const api = createElectronApi({ + invoke: async (channel: string, payload?: unknown) => { + calls.push({ channel, payload }) + return { success: true, entries: [] } + }, + send: () => undefined, + on: () => undefined, + removeAllListeners: () => undefined, + }, { setZoomFactor: () => undefined }) + + await api.workspace.library.list() + await api.workspace.library.read({ + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + sourceWorkspacePath: 'Workflows/checkpoints/hero.glb', + }) + await api.workspace.library.open({ + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + sourceWorkspacePath: 'Workflows/checkpoints/hero.glb', + }) + + assert.deepEqual(calls, [ + { channel: 'workspace:library:list', payload: undefined }, + { + channel: 'workspace:library:read', + payload: { + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + sourceWorkspacePath: 'Workflows/checkpoints/hero.glb', + }, + }, + { + channel: 'workspace:library:open', + payload: { + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + sourceWorkspacePath: 'Workflows/checkpoints/hero.glb', + }, + }, + ]) +}) diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts new file mode 100644 index 0000000..6ced29c --- /dev/null +++ b/electron/preload/electron-api.ts @@ -0,0 +1,292 @@ +import type { + AssetLibraryListResult, + AssetLibraryOpenRequest, + AssetLibraryOpenResult, + AssetLibraryReadRequest, + AssetLibraryReadResult, +} from '../../src/shared/types/assetLibrary' + +export interface IpcRendererLike { + invoke(channel: string, ...args: unknown[]): Promise + send(channel: string, ...args: unknown[]): void + on(channel: string, listener: (...args: unknown[]) => void): void + removeAllListeners(channel: string): void +} + +export interface WebFrameLike { + setZoomFactor(factor: number): void +} + +export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFrameLike) { + return { + // Window controls + window: { + minimize: () => ipcRenderer.send('window:minimize'), + maximize: () => ipcRenderer.send('window:maximize'), + close: () => ipcRenderer.send('window:close'), + }, + + // Renderer UI (zoom whole page β€” scales every px/rem consistently) + ui: { setZoomFactor: (factor: number) => webFrame.setZoomFactor(factor) }, + + // Shell utilities + shell: { openExternal: (url: string) => ipcRenderer.invoke('shell:openExternal', url) }, + + // System info + system: { + memory: (): Promise<{ total: number; used: number; available: number }> => + ipcRenderer.invoke('system:memory') as Promise<{ total: number; used: number; available: number }>, + }, + + // Python / FastAPI bridge + python: { + start: (): Promise<{ success: boolean; port?: number; error?: string }> => + ipcRenderer.invoke('python:start') as Promise<{ success: boolean; port?: number; error?: string }>, + status: (): Promise<{ ready: boolean; apiUrl: string }> => + ipcRenderer.invoke('python:status') as Promise<{ ready: boolean; apiUrl: string }>, + onCrashed: (cb: (data: { code: number | null }) => void) => { + ipcRenderer.on('python:crashed', (_event, data) => cb(data as { code: number | null })) + }, + offCrashed: () => ipcRenderer.removeAllListeners('python:crashed'), + onLog: (cb: (line: string) => void) => { + ipcRenderer.on('python:log', (_event, line) => cb(line as string)) + }, + offLog: () => ipcRenderer.removeAllListeners('python:log'), + }, + + // File system dialogs + local file reading + fs: { + selectImage: (): Promise => + ipcRenderer.invoke('fs:selectImage') as Promise, + selectMeshFile: (): Promise => + ipcRenderer.invoke('fs:selectMeshFile') as Promise, + saveModel: (defaultName: string): Promise => + ipcRenderer.invoke('fs:saveModel', defaultName) as Promise, + readFileBase64: (filePath: string): Promise => + ipcRenderer.invoke('fs:readFileBase64', filePath) as Promise, + selectDirectory: (defaultPath?: string): Promise => + ipcRenderer.invoke('fs:selectDirectory', defaultPath) as Promise, + savePath: (args: { filters: { name: string; extensions: string[] }[]; defaultPath?: string }): Promise => + ipcRenderer.invoke('fs:savePath', args) as Promise, + listDir: (dirPath: string): Promise => + ipcRenderer.invoke('fs:listDir', dirPath) as Promise, + moveDirectory: (args: { src: string; dest: string }): Promise<{ success: boolean; error?: string }> => + ipcRenderer.invoke('fs:moveDirectory', args) as Promise<{ success: boolean; error?: string }>, + deleteDirectory: (dirPath: string): Promise<{ success: boolean; error?: string }> => + ipcRenderer.invoke('fs:deleteDirectory', dirPath) as Promise<{ success: boolean; error?: string }>, + readScreenshotDataUrl: (filename: string): Promise => + ipcRenderer.invoke('fs:readScreenshotDataUrl', filename) as Promise, + }, + + // Settings + settings: { + get: (): Promise<{ modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string; hfToken?: string }> => + ipcRenderer.invoke('settings:get') as Promise<{ modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string; hfToken?: string }>, + set: (patch: { modelsDir?: string; workspaceDir?: string; workflowsDir?: string; extensionsDir?: string; hfToken?: string }): Promise<{ modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string; hfToken?: string }> => + ipcRenderer.invoke('settings:set', patch) as Promise<{ modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string; hfToken?: string }>, + }, + + // Cache + cache: { + clear: (): Promise<{ success: boolean; error?: string }> => + ipcRenderer.invoke('cache:clear') as Promise<{ success: boolean; error?: string }>, + }, + + // API helpers (calls FastAPI from the main process) + api: { + updatePaths: (patch: { modelsDir?: string; workspaceDir?: string; extensionsDir?: string }): Promise<{ success: boolean; error?: string }> => + ipcRenderer.invoke('api:updatePaths', patch) as Promise<{ success: boolean; error?: string }>, + }, + + // Model management + model: { + export: (args: { outputUrl: string; format: string }) => ipcRenderer.invoke('model:export', args), + listDownloaded: () => ipcRenderer.invoke('model:listDownloaded'), + isDownloaded: (modelId: string, downloadCheck?: string) => ipcRenderer.invoke('model:isDownloaded', modelId, downloadCheck), + download: (repoId: string, modelId: string, skipPrefixes?: string[], includePrefixes?: string[]) => + ipcRenderer.invoke('model:download', { repoId, modelId, skipPrefixes, includePrefixes }), + pauseDownload: (modelId: string) => ipcRenderer.invoke('model:pauseDownload', modelId), + cancelDownload: (modelId: string) => ipcRenderer.invoke('model:cancelDownload', modelId), + delete: (modelId: string) => ipcRenderer.invoke('model:delete', modelId), + unloadAll: () => ipcRenderer.invoke('model:unloadAll'), + showInFolder: (modelId: string) => ipcRenderer.invoke('model:showInFolder', modelId), + activeDownloads: (): Promise<{ modelId: string; percent: number; file?: string; fileIndex?: number; totalFiles?: number }[]> => + ipcRenderer.invoke('model:activeDownloads') as Promise<{ modelId: string; percent: number; file?: string; fileIndex?: number; totalFiles?: number }[]>, + onProgress: (cb: (data: { + modelId: string + percent: number + file?: string + fileIndex?: number + totalFiles?: number + status?: string + bytesDownloaded?: number + totalBytes?: number + stalledSeconds?: number + paused?: boolean + cancelled?: boolean + }) => void) => { + ipcRenderer.on('model:downloadProgress', (_event, data) => cb(data as { + modelId: string + percent: number + file?: string + fileIndex?: number + totalFiles?: number + status?: string + bytesDownloaded?: number + totalBytes?: number + stalledSeconds?: number + paused?: boolean + cancelled?: boolean + })) + }, + offProgress: () => ipcRenderer.removeAllListeners('model:downloadProgress'), + }, + + // App metadata + app: { + info: (): Promise<{ version: string; userData: string; modelsDir: string; apiUrl: string; platform: string; arch: string }> => + ipcRenderer.invoke('app:info') as Promise<{ version: string; userData: string; modelsDir: string; apiUrl: string; platform: string; arch: string }>, + onError: (cb: (message: string) => void) => { + ipcRenderer.on('app:error', (_event, message) => cb(message as string)) + }, + offError: () => ipcRenderer.removeAllListeners('app:error'), + }, + + // Logging + log: { + error: (message: string) => ipcRenderer.send('log:error', message), + getPath: (): Promise => ipcRenderer.invoke('log:getPath') as Promise, + readAll: (session?: string): Promise> => ipcRenderer.invoke('log:readAll', session) as Promise>, + listSessions: (): Promise => ipcRenderer.invoke('log:listSessions') as Promise, + }, + + // Workspace filesystem-based persistence + workspace: { + listCollections: (): Promise => + ipcRenderer.invoke('workspace:listCollections') as Promise, + createCollection: (name: string): Promise => + ipcRenderer.invoke('workspace:createCollection', name) as Promise, + renameCollection: (oldName: string, newName: string): Promise => + ipcRenderer.invoke('workspace:renameCollection', { oldName, newName }) as Promise, + deleteCollection: (name: string): Promise => + ipcRenderer.invoke('workspace:deleteCollection', name) as Promise, + listJobs: (collection: string): Promise => + ipcRenderer.invoke('workspace:listJobs', collection) as Promise, + saveJobMeta: (collection: string, filename: string, meta: unknown): Promise => + ipcRenderer.invoke('workspace:saveJobMeta', { collection, filename, meta }) as Promise, + deleteJob: (collection: string, filename: string): Promise => + ipcRenderer.invoke('workspace:deleteJob', { collection, filename }) as Promise, + library: { + list: (): Promise => ipcRenderer.invoke('workspace:library:list') as Promise, + read: (request: AssetLibraryReadRequest): Promise => ipcRenderer.invoke('workspace:library:read', request) as Promise, + open: (request: AssetLibraryOpenRequest): Promise => ipcRenderer.invoke('workspace:library:open', request) as Promise, + }, + }, + + // Extensions + extensions: { + list: (): Promise => + ipcRenderer.invoke('extensions:list') as Promise, + + installFromGitHub: (url: string): Promise<{ + success: boolean; error?: string; cancelled?: boolean + extensionId?: string + extension?: unknown + }> => ipcRenderer.invoke('extensions:installFromGitHub', url) as Promise<{ + success: boolean; error?: string; cancelled?: boolean + extensionId?: string + extension?: unknown + }>, + + installFromLocal: (): Promise<{ + success: boolean; error?: string; cancelled?: boolean + extensionId?: string + extension?: unknown + localPath?: string + }> => ipcRenderer.invoke('extensions:installFromLocal') as Promise<{ + success: boolean; error?: string; cancelled?: boolean + extensionId?: string + extension?: unknown + localPath?: string + }>, + + uninstall: (extensionId: string): Promise<{ success: boolean; error?: string }> => + ipcRenderer.invoke('extensions:uninstall', extensionId) as Promise<{ success: boolean; error?: string }>, + + repair: (extensionId: string): Promise<{ success: boolean; error?: string }> => + ipcRenderer.invoke('extensions:repair', extensionId) as Promise<{ success: boolean; error?: string }>, + + reload: (): Promise<{ success: boolean; error?: string }> => + ipcRenderer.invoke('extensions:reload') as Promise<{ success: boolean; error?: string }>, + + runProcess: ( + extensionId: string, + input: { filePath?: string; text?: string; nodeId?: string }, + params: Record, + ): Promise<{ success: boolean; result?: { filePath?: string; text?: string }; error?: string }> => + ipcRenderer.invoke('extensions:runProcess', extensionId, input, params) as Promise<{ success: boolean; result?: { filePath?: string; text?: string }; error?: string }>, + + onInstallProgress: (cb: (data: { + step: 'downloading' | 'extracting' | 'validating' | 'setting_up' | 'done' | 'error' + percent?: number + extensionId?: string + message?: string + }) => void) => { + ipcRenderer.on('extensions:installProgress', (_event, data) => cb(data as { + step: 'downloading' | 'extracting' | 'validating' | 'setting_up' | 'done' | 'error' + percent?: number + extensionId?: string + message?: string + })) + }, + offInstallProgress: () => ipcRenderer.removeAllListeners('extensions:installProgress'), + }, + + // Workflows + workflows: { + list: (): Promise => ipcRenderer.invoke('workflows:list') as Promise, + save: (workflow: { id: string; [key: string]: unknown }): Promise<{ success: boolean; error?: string }> => ipcRenderer.invoke('workflows:save', workflow) as Promise<{ success: boolean; error?: string }>, + delete: (id: string): Promise<{ success: boolean; error?: string }> => ipcRenderer.invoke('workflows:delete', id) as Promise<{ success: boolean; error?: string }>, + import: (): Promise<{ success: boolean; error?: string; workflow?: unknown }> => ipcRenderer.invoke('workflows:import') as Promise<{ success: boolean; error?: string; workflow?: unknown }>, + export: (workflow: { id: string; name?: string; [key: string]: unknown }): Promise<{ success: boolean; error?: string }> => ipcRenderer.invoke('workflows:export', workflow) as Promise<{ success: boolean; error?: string }>, + }, + + // Auto-updater + updater: { + check: (): Promise<{ success: boolean }> => + ipcRenderer.invoke('updater:check') as Promise<{ success: boolean }>, + quitAndInstall: (): Promise => + ipcRenderer.invoke('updater:quitAndInstall') as Promise, + onApplying: (cb: (data: { version: string }) => void) => { + ipcRenderer.on('updater:applying', (_event, data) => cb(data as { version: string })) + }, + offApplying: () => ipcRenderer.removeAllListeners('updater:applying'), + onMajorMinorAvailable: (cb: (data: { version: string }) => void) => { + ipcRenderer.on('updater:major-minor-available', (_event, data) => cb(data as { version: string })) + }, + offMajorMinorAvailable: () => ipcRenderer.removeAllListeners('updater:major-minor-available'), + }, + + // First-run setup + setup: { + check: (): Promise<{ needed: boolean; defaultDataDir: string; platform: string; arch: string }> => + ipcRenderer.invoke('setup:check') as Promise<{ needed: boolean; defaultDataDir: string; platform: string; arch: string }>, + run: (): Promise<{ success: boolean; error?: string }> => + ipcRenderer.invoke('setup:run') as Promise<{ success: boolean; error?: string }>, + saveDataDir: (baseDir: string): Promise => + ipcRenderer.invoke('setup:saveDataDir', { baseDir }) as Promise, + onProgress: (cb: (data: { step: string; percent: number; currentPackage?: string }) => void) => { + ipcRenderer.on('setup:progress', (_event, data) => cb(data as { step: string; percent: number; currentPackage?: string })) + }, + offProgress: () => ipcRenderer.removeAllListeners('setup:progress'), + onComplete: (cb: () => void) => { + ipcRenderer.on('setup:complete', () => cb()) + }, + offComplete: () => ipcRenderer.removeAllListeners('setup:complete'), + onError: (cb: (data: { message: string }) => void) => { + ipcRenderer.on('setup:error', (_event, data) => cb(data as { message: string })) + }, + offError: () => ipcRenderer.removeAllListeners('setup:error'), + }, + } +} diff --git a/electron/preload/index.ts b/electron/preload/index.ts index c5e455b..ec34ebf 100644 --- a/electron/preload/index.ts +++ b/electron/preload/index.ts @@ -1,213 +1,6 @@ -import { contextBridge, ipcRenderer } from 'electron' +import { contextBridge, ipcRenderer, webFrame } from 'electron' -// Expose a typed API to the renderer process via window.electron -contextBridge.exposeInMainWorld('electron', { - // Window controls - window: { - minimize: () => ipcRenderer.send('window:minimize'), - maximize: () => ipcRenderer.send('window:maximize'), - close: () => ipcRenderer.send('window:close') - }, - - // Shell utilities - shell: { - openExternal: (url: string) => ipcRenderer.invoke('shell:openExternal', url), - }, - - // Python / FastAPI bridge - python: { - start: (): Promise<{ success: boolean; port?: number; error?: string }> => - ipcRenderer.invoke('python:start'), - status: (): Promise<{ ready: boolean; apiUrl: string }> => - ipcRenderer.invoke('python:status'), - onCrashed: (cb: (data: { code: number | null }) => void) => { - ipcRenderer.on('python:crashed', (_event, data) => cb(data)) - }, - offCrashed: () => ipcRenderer.removeAllListeners('python:crashed'), - onLog: (cb: (line: string) => void) => { - ipcRenderer.on('python:log', (_event, line) => cb(line)) - }, - offLog: () => ipcRenderer.removeAllListeners('python:log') - }, - - // File system dialogs + local file reading - fs: { - selectImage: (): Promise => - ipcRenderer.invoke('fs:selectImage'), - selectMeshFile: (): Promise => - ipcRenderer.invoke('fs:selectMeshFile'), - saveModel: (defaultName: string): Promise => - ipcRenderer.invoke('fs:saveModel', defaultName), - readFileBase64: (filePath: string): Promise => - ipcRenderer.invoke('fs:readFileBase64', filePath), - selectDirectory: (): Promise => - ipcRenderer.invoke('fs:selectDirectory'), - savePath: (args: { filters: { name: string; extensions: string[] }[]; defaultPath?: string }): Promise => - ipcRenderer.invoke('fs:savePath', args), - listDir: (dirPath: string): Promise => - ipcRenderer.invoke('fs:listDir', dirPath), - moveDirectory: (args: { src: string; dest: string }): Promise<{ success: boolean; error?: string }> => - ipcRenderer.invoke('fs:moveDirectory', args), - deleteDirectory: (dirPath: string): Promise<{ success: boolean; error?: string }> => - ipcRenderer.invoke('fs:deleteDirectory', dirPath), - readScreenshotDataUrl: (filename: string): Promise => - ipcRenderer.invoke('fs:readScreenshotDataUrl', filename), - }, - - // Settings - settings: { - get: (): Promise<{ modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string; hfToken?: string }> => - ipcRenderer.invoke('settings:get'), - set: (patch: { modelsDir?: string; workspaceDir?: string; workflowsDir?: string; extensionsDir?: string; hfToken?: string }): Promise<{ modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string; hfToken?: string }> => - ipcRenderer.invoke('settings:set', patch), - }, - - // Cache - cache: { - clear: (): Promise<{ success: boolean; error?: string }> => - ipcRenderer.invoke('cache:clear'), - }, - - // API helpers (calls FastAPI from the main process) - api: { - updatePaths: (patch: { modelsDir?: string; workspaceDir?: string; extensionsDir?: string }): Promise<{ success: boolean; error?: string }> => - ipcRenderer.invoke('api:updatePaths', patch), - }, - - // Model management - model: { - export: (args: { outputUrl: string; format: string }) => ipcRenderer.invoke('model:export', args), - listDownloaded: () => ipcRenderer.invoke('model:listDownloaded'), - isDownloaded: (modelId: string) => ipcRenderer.invoke('model:isDownloaded', modelId), - download: (repoId: string, modelId: string, skipPrefixes?: string[]) => ipcRenderer.invoke('model:download', { repoId, modelId, skipPrefixes }), - delete: (modelId: string) => ipcRenderer.invoke('model:delete', modelId), - unloadAll: () => ipcRenderer.invoke('model:unloadAll'), - showInFolder: (modelId: string) => ipcRenderer.invoke('model:showInFolder', modelId), - activeDownloads: (): Promise<{ modelId: string; percent: number; file?: string; fileIndex?: number; totalFiles?: number }[]> => ipcRenderer.invoke('model:activeDownloads'), - onProgress: (cb: (data: { modelId: string; percent: number; file?: string; fileIndex?: number; totalFiles?: number; status?: string }) => void) => { - ipcRenderer.on('model:downloadProgress', (_event, data) => cb(data)) - }, - offProgress: () => ipcRenderer.removeAllListeners('model:downloadProgress') - }, - - // App metadata - app: { - info: (): Promise<{ version: string; userData: string; modelsDir: string; apiUrl: string }> => - ipcRenderer.invoke('app:info'), - onError: (cb: (message: string) => void) => { - ipcRenderer.on('app:error', (_event, message) => cb(message)) - }, - offError: () => ipcRenderer.removeAllListeners('app:error'), - }, - - // Logging - log: { - error: (message: string) => ipcRenderer.send('log:error', message), - getPath: (): Promise => ipcRenderer.invoke('log:getPath'), - readAll: (session?: string): Promise> => ipcRenderer.invoke('log:readAll', session), - listSessions: (): Promise => ipcRenderer.invoke('log:listSessions'), - }, - - // Workspace filesystem-based persistence - workspace: { - listCollections: (): Promise => - ipcRenderer.invoke('workspace:listCollections'), - createCollection: (name: string): Promise => - ipcRenderer.invoke('workspace:createCollection', name), - renameCollection: (oldName: string, newName: string): Promise => - ipcRenderer.invoke('workspace:renameCollection', { oldName, newName }), - deleteCollection: (name: string): Promise => - ipcRenderer.invoke('workspace:deleteCollection', name), - listJobs: (collection: string): Promise => - ipcRenderer.invoke('workspace:listJobs', collection), - saveJobMeta: (collection: string, filename: string, meta: unknown): Promise => - ipcRenderer.invoke('workspace:saveJobMeta', { collection, filename, meta }), - deleteJob: (collection: string, filename: string): Promise => - ipcRenderer.invoke('workspace:deleteJob', { collection, filename }), - }, +import { createElectronApi } from './electron-api' - // Extensions - extensions: { - list: (): Promise => - ipcRenderer.invoke('extensions:list'), - - installFromGitHub: (url: string): Promise<{ - success: boolean; error?: string - extensionId?: string - extension?: unknown - }> => ipcRenderer.invoke('extensions:installFromGitHub', url), - - uninstall: (extensionId: string): Promise<{ success: boolean; error?: string }> => - ipcRenderer.invoke('extensions:uninstall', extensionId), - - repair: (extensionId: string): Promise<{ success: boolean; error?: string }> => - ipcRenderer.invoke('extensions:repair', extensionId), - - reload: (): Promise<{ success: boolean; error?: string }> => - ipcRenderer.invoke('extensions:reload'), - - runProcess: ( - extensionId: string, - input: { filePath?: string; text?: string; nodeId?: string }, - params: Record, - ): Promise<{ success: boolean; result?: { filePath?: string; text?: string }; error?: string }> => - ipcRenderer.invoke('extensions:runProcess', extensionId, input, params), - - onInstallProgress: (cb: (data: { - step: 'downloading' | 'extracting' | 'validating' | 'setting_up' | 'done' | 'error' - percent?: number - extensionId?: string - message?: string - }) => void) => { - ipcRenderer.on('extensions:installProgress', (_event, data) => cb(data)) - }, - offInstallProgress: () => ipcRenderer.removeAllListeners('extensions:installProgress'), - }, - - // Workflows - workflows: { - list: (): Promise => ipcRenderer.invoke('workflows:list'), - save: (workflow: { id: string; [key: string]: unknown }): Promise<{ success: boolean; error?: string }> => ipcRenderer.invoke('workflows:save', workflow), - delete: (id: string): Promise<{ success: boolean; error?: string }> => ipcRenderer.invoke('workflows:delete', id), - import: (): Promise<{ success: boolean; error?: string; workflow?: unknown }> => ipcRenderer.invoke('workflows:import'), - export: (workflow: { id: string; name?: string; [key: string]: unknown }): Promise<{ success: boolean; error?: string }> => ipcRenderer.invoke('workflows:export', workflow), - }, - - // Auto-updater - updater: { - check: (): Promise<{ success: boolean }> => - ipcRenderer.invoke('updater:check'), - quitAndInstall: (): Promise => - ipcRenderer.invoke('updater:quitAndInstall'), - onApplying: (cb: (data: { version: string }) => void) => { - ipcRenderer.on('updater:applying', (_event, data) => cb(data)) - }, - offApplying: () => ipcRenderer.removeAllListeners('updater:applying'), - onMajorMinorAvailable: (cb: (data: { version: string }) => void) => { - ipcRenderer.on('updater:major-minor-available', (_event, data) => cb(data)) - }, - offMajorMinorAvailable: () => ipcRenderer.removeAllListeners('updater:major-minor-available'), - }, - - // First-run setup - setup: { - check: (): Promise<{ needed: boolean; defaultDataDir: string }> => - ipcRenderer.invoke('setup:check'), - run: (): Promise<{ success: boolean; error?: string }> => - ipcRenderer.invoke('setup:run'), - saveDataDir: (baseDir: string): Promise => - ipcRenderer.invoke('setup:saveDataDir', { baseDir }), - onProgress: (cb: (data: { step: string; percent: number; currentPackage?: string }) => void) => { - ipcRenderer.on('setup:progress', (_e, data) => cb(data)) - }, - offProgress: () => ipcRenderer.removeAllListeners('setup:progress'), - onComplete: (cb: () => void) => { - ipcRenderer.on('setup:complete', () => cb()) - }, - offComplete: () => ipcRenderer.removeAllListeners('setup:complete'), - onError: (cb: (data: { message: string }) => void) => { - ipcRenderer.on('setup:error', (_e, data) => cb(data)) - }, - offError: () => ipcRenderer.removeAllListeners('setup:error'), - } -}) +// Expose a typed API to the renderer process via window.electron +contextBridge.exposeInMainWorld('electron', createElectronApi(ipcRenderer, webFrame)) diff --git a/package-lock.json b/package-lock.json index 28cca19..1f336a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,15 @@ { "name": "modly", - "version": "0.3.0", + "version": "0.3.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "modly", - "version": "0.3.0", + "version": "0.3.5", "dependencies": { "@electron-toolkit/utils": "^4.0.0", + "@mkkellogg/gaussian-splats-3d": "^0.4.7", "@react-three/drei": "^9.120.0", "@react-three/fiber": "^8.17.10", "@react-three/postprocessing": "^2.19.1", @@ -1536,6 +1537,15 @@ "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz", "integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==" }, + "node_modules/@mkkellogg/gaussian-splats-3d": { + "version": "0.4.7", + "resolved": "https://registry.npmjs.org/@mkkellogg/gaussian-splats-3d/-/gaussian-splats-3d-0.4.7.tgz", + "integrity": "sha512-0vy9/i9sJLFH/v3WJZ4axCsqjkToe8UsV3xY7bvK5EUC0akiRsWZODoCiSzpxhTLNyzSKTsyQKozIFeNA5RWRA==", + "license": "MIT", + "peerDependencies": { + "three": ">=0.160.0" + } + }, "node_modules/@monogrid/gainmap-js": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/@monogrid/gainmap-js/-/gainmap-js-3.4.0.tgz", diff --git a/package.json b/package.json index c323a0c..e4f205e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "modly", - "version": "0.3.6", + "version": "0.4.0", "description": "Local AI-powered 3D mesh generation from images", "main": "./out/main/index.js", "author": "Modly", @@ -10,11 +10,14 @@ "build": "node scripts/build-builtins.mjs && electron-vite build", "preview": "electron-vite preview", "prepare-resources": "node scripts/download-python-embed.js", + "test": "cd api && python3 -m unittest discover -s tests && cd .. && node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", + "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." }, "dependencies": { "@electron-toolkit/utils": "^4.0.0", + "@mkkellogg/gaussian-splats-3d": "^0.4.7", "@react-three/drei": "^9.120.0", "@react-three/fiber": "^8.17.10", "@react-three/postprocessing": "^2.19.1", @@ -63,7 +66,10 @@ "filter": [ "**/*", "!.venv/**/*", - "!__pycache__/**/*" + "!__pycache__/**/*", + "!**/build/**/*", + "!**/*.pyd", + "!**/*.egg-info/**/*" ] }, { @@ -93,7 +99,15 @@ }, "mac": { "target": "dmg", - "icon": "resources/icons/icon.icns" + "icon": "resources/icons/icon.icns", + "artifactName": "${productName}-${version}-arm64.${ext}", + "category": "public.app-category.graphics-design", + "extraResources": [ + { + "from": "resources/python-embed", + "to": "python-embed" + } + ] }, "linux": { "target": "AppImage", diff --git a/scripts/node-ts-extensionless-loader.mjs b/scripts/node-ts-extensionless-loader.mjs new file mode 100644 index 0000000..b916490 --- /dev/null +++ b/scripts/node-ts-extensionless-loader.mjs @@ -0,0 +1,14 @@ +export async function resolve(specifier, context, nextResolve) { + try { + return await nextResolve(specifier, context) + } catch (error) { + if (!shouldRetryWithTypeScriptExtension(specifier, error)) throw error + return nextResolve(`${specifier}.ts`, context) + } +} + +function shouldRetryWithTypeScriptExtension(specifier, error) { + return error?.code === 'ERR_MODULE_NOT_FOUND' + && (specifier.startsWith('./') || specifier.startsWith('../')) + && !specifier.endsWith('.ts') +} diff --git a/src/App.tsx b/src/App.tsx index 82180ee..6409b76 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,12 +1,15 @@ -import { useEffect, useState } from 'react' -import { useAppStore } from '@shared/stores/appStore' +import { useEffect, useLayoutEffect, useState } from 'react' +import { useAppStore, type UiScale } from '@shared/stores/appStore' import FirstRunSetup from '@areas/setup/FirstRunSetup' import MainLayout from '@shared/components/layout/MainLayout' import { UpdateModal } from '@shared/components/ui/UpdateModal' import { ErrorModal } from '@shared/components/ui/ErrorModal' +import { Toast } from '@shared/components/ui/Toast' + +const UI_SCALE_FACTORS: Record = { small: 0.875, medium: 1, large: 1.25 } export default function App(): JSX.Element { - const { checkSetup, setupStatus, initApp, backendStatus, showError } = useAppStore() + const { checkSetup, setupStatus, initApp, backendStatus, showError, useAtkinsonFont, uiScale } = useAppStore() const [updateVersion, setUpdateVersion] = useState(null) const [currentVersion, setCurrentVersion] = useState('') @@ -22,6 +25,17 @@ export default function App(): JSX.Element { } }, []) + // Apply before paint to avoid a flash of default font/size on launch. + useLayoutEffect(() => { + document.documentElement.style.setProperty( + '--app-font', + useAtkinsonFont + ? "'Atkinson Hyperlegible', system-ui, sans-serif" + : "'Inter', system-ui, sans-serif" + ) + window.electron.ui.setZoomFactor(UI_SCALE_FACTORS[uiScale]) + }, [useAtkinsonFont, uiScale]) + useEffect(() => { if (setupStatus === 'done') initApp() }, [setupStatus]) @@ -41,12 +55,14 @@ export default function App(): JSX.Element { onDismiss={() => setUpdateVersion(null)} /> )} + ) return ( <> + ) diff --git a/src/areas/generate/GeneratePage.tsx b/src/areas/generate/GeneratePage.tsx index 318d235..643c7e1 100644 --- a/src/areas/generate/GeneratePage.tsx +++ b/src/areas/generate/GeneratePage.tsx @@ -1,11 +1,27 @@ -import { useState, useRef, useCallback, useEffect } from 'react' -import { useAppStore } from '@shared/stores/appStore' -import type { GenerationJob } from '@shared/stores/appStore' +import { useState, useRef, useCallback, useEffect, useMemo } from 'react' +import type { ReactNode } from 'react' +import { useAppStore, DEFAULT_LIGHT_SETTINGS } from '@shared/stores/appStore' +import type { GenerationJob, LightSettings } from '@shared/stores/appStore' import { useApi } from '@shared/hooks/useApi' import { ColorPicker } from '@shared/components/ui' import GenerationHUD from './components/GenerationHUD' import Viewer3D from './components/Viewer3D' import WorkflowPanel from './components/WorkflowPanel' +import { getDefaultAssetLibraryService } from './assetLibraryService' +import { resolveAssetLibraryOpenTarget, type ProjectedAssetLibraryEntry } from './assetLibraryProjection' +import { + ASSET_LIBRARY_SORT_OPTIONS, + buildAssetLibraryOpenRequest, + createAssetLibraryOpenJob, + describeAssetLibraryOpenability, + filterAssetLibraryScopeGroups, + getDefaultAssetLibraryCollapsedSectionKeys, + isAssetLibraryEntryOpenable, + resolveOpenPanelAfterLibrarySelection, + toggleAssetLibrarySectionKey, + type AssetLibrarySortMode, + type GenerateOpenPanel, +} from './assetLibraryUi' const MIN_WIDTH = 220 const MAX_WIDTH = 520 @@ -45,6 +61,38 @@ function ExportDropdown({ ) } +// --------------------------------------------------------------------------- +// ToolButton β€” icon-only toolbar button with tooltip + active state +// --------------------------------------------------------------------------- + +function ToolButton({ + label, + active, + onClick, + children, +}: { + label: string + active: boolean + onClick: () => void + children: ReactNode +}) { + return ( + + ) +} + // --------------------------------------------------------------------------- // Decimate popover // --------------------------------------------------------------------------- @@ -127,20 +175,6 @@ function DecimatePopover({ // Light popover // --------------------------------------------------------------------------- -export interface LightSettings { - mainIntensity: number - mainColor: string - fillIntensity: number - fillColor: string -} - -export const DEFAULT_LIGHT_SETTINGS: LightSettings = { - mainIntensity: 1.5, - mainColor: '#ffffff', - fillIntensity: 0.6, - fillColor: '#ffffff', -} - function LightPopover({ settings, onChange, @@ -181,6 +215,27 @@ function LightPopover({ ) } + function plainRow(label: string, intensityKey: keyof LightSettings, max: number) { + const value = (settings[intensityKey] as number) ?? (DEFAULT_LIGHT_SETTINGS[intensityKey] as number) + return ( +
+
+ {label} + {value.toFixed(2)} +
+ onChange({ ...settings, [intensityKey]: parseFloat(e.target.value) })} + className="w-full h-1.5 accent-violet-500 cursor-pointer" + /> +
+ ) + } + return (
@@ -194,6 +249,8 @@ function LightPopover({
{lightRow('Sun', 'mainColor', 'mainIntensity', 4)} {lightRow('Fill', 'fillColor', 'fillIntensity', 2)} + {plainRow('Ambient', 'ambientIntensity', 1.5)} + {plainRow('Environment', 'envIntensity', 2)} + ) +} + +function AssetLibraryPopover({ + entries, + selectedEntryId, + loading, + opening, + error, + searchQuery, + sortMode, + collapsedSectionKeys, + onSelectEntry, + onSearchQueryChange, + onSortModeChange, + onToggleSection, + onOpenSelected, + onRefresh, + onClose, +}: { + entries: ProjectedAssetLibraryEntry[] + selectedEntryId: string | null + loading: boolean + opening: boolean + error: string | null + searchQuery: string + sortMode: AssetLibrarySortMode + collapsedSectionKeys: string[] + onSelectEntry: (entryId: string) => void + onSearchQueryChange: (value: string) => void + onSortModeChange: (value: AssetLibrarySortMode) => void + onToggleSection: (sectionKey: string) => void + onOpenSelected: () => void + onRefresh: () => void + onClose: () => void +}) { + const scopeGroups = filterAssetLibraryScopeGroups(entries, searchQuery, sortMode) + const visibleEntryIds = new Set(scopeGroups.flatMap((scopeGroup) => scopeGroup.entryGroups.flatMap((group) => group.entries.map((entry) => entry.id)))) + const selectedEntry = selectedEntryId && visibleEntryIds.has(selectedEntryId) + ? entries.find((entry) => entry.id === selectedEntryId) ?? null + : null + const normalizedSearchQuery = searchQuery.trim() + const openDisabled = !selectedEntry || !isAssetLibraryEntryOpenable(selectedEntry) || loading || opening + const selectedMessage = selectedEntry + ? describeAssetLibraryOpenability(selectedEntry) + : scopeGroups.length === 0 && normalizedSearchQuery + ? `No workspace assets match β€œ${normalizedSearchQuery}”.` + : 'Select an asset to open it in Generate.' + + return ( +
+
+
+

Workspace library

+

Select a workspace asset and open the supported source in Generate.

+
+ +
+ + + +
+
+ + onSearchQueryChange(event.target.value)} + placeholder="Search by name, path, scope, or capability" + className="bg-zinc-800 border border-zinc-700 rounded-lg px-2.5 py-1.5 text-xs text-zinc-200 w-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-violet-400" + /> +
+
+ + +
+
+ + {loading ? ( +

Loading workspace assets…

+ ) : scopeGroups.length === 0 && !normalizedSearchQuery ? ( +

No workspace assets are indexed yet.

+ ) : scopeGroups.length === 0 ? ( +

{`No workspace assets match β€œ${normalizedSearchQuery}”.`}

+ ) : ( +
+ {scopeGroups.map((scopeGroup) => { + const scopeExpanded = !collapsedSectionKeys.includes(scopeGroup.sectionKey) + const scopeRegionId = `asset-library-${scopeGroup.sectionKey.replace(/[^a-z0-9-]+/gi, '-')}` + return ( +
+ + {scopeExpanded && ( +
+ {scopeGroup.entryGroups.map((group) => { + const capabilityExpanded = !collapsedSectionKeys.includes(group.sectionKey) + const capabilityRegionId = `asset-library-${group.sectionKey.replace(/[^a-z0-9-]+/gi, '-')}` + return ( +
+ + {capabilityExpanded && ( +
+ {group.entries.map((entry) => { + const selected = entry.id === selectedEntryId + return ( + + ) + })} +
+ )} +
+ ) + })} +
+ )} +
+ ) + })} +
+ )} + +
+

{selectedMessage}

+ {error &&

{error}

} +
+ + +
+ ) +} + // --------------------------------------------------------------------------- // GeneratePage // --------------------------------------------------------------------------- @@ -273,32 +562,48 @@ function SmoothPopover({ export default function GeneratePage(): JSX.Element { const [unloadStatus, setUnloadStatus] = useState<'idle' | 'done'>('idle') const [panelWidth, setPanelWidth] = useState(DEFAULT_WIDTH) - const [openPanel, setOpenPanel] = useState<'export' | 'decimate' | 'smooth' | 'import' | 'light' | null>(null) - const [lightSettings, setLightSettings] = useState(DEFAULT_LIGHT_SETTINGS) + const [openPanel, setOpenPanel] = useState(null) const [decimating, setDecimating] = useState(false) const [smoothing, setSmoothing] = useState(false) const [importing, setImporting] = useState(false) + const [libraryEntries, setLibraryEntries] = useState([]) + const [librarySelectedEntryId, setLibrarySelectedEntryId] = useState(null) + const [libraryLoaded, setLibraryLoaded] = useState(false) + const [libraryLoading, setLibraryLoading] = useState(false) + const [libraryOpening, setLibraryOpening] = useState(false) + const [libraryError, setLibraryError] = useState(null) + const [librarySearchQuery, setLibrarySearchQuery] = useState('') + const [librarySortMode, setLibrarySortMode] = useState('type') + const [libraryCollapsedSectionKeys, setLibraryCollapsedSectionKeys] = useState(() => getDefaultAssetLibraryCollapsedSectionKeys()) + const [gizmoMode, setGizmoMode] = useState<'translate' | 'rotate' | 'scale' | null>(null) const dragging = useRef(false) + // Populated by Viewer3D β€” undoes the latest live gizmo transform, if any. + const gizmoUndoRef = useRef<(() => boolean) | null>(null) + const lightSettings = useAppStore((s) => s.lightSettings) + const setLightSettings = useAppStore((s) => s.setLightSettings) const isGenerating = useAppStore((s) => s.currentJob?.status === 'uploading' || s.currentJob?.status === 'generating' ) const currentJob = useAppStore((s) => s.currentJob) const apiUrl = useAppStore((s) => s.apiUrl) + const showError = useAppStore((s) => s.showError) const updateCurrentJob = useAppStore((s) => s.updateCurrentJob) const setCurrentJob = useAppStore((s) => s.setCurrentJob) const meshStats = useAppStore((s) => s.meshStats) + const meshSelected = useAppStore((s) => s.meshSelected) const pushMeshUrl = useAppStore((s) => s.pushMeshUrl) const undoMesh = useAppStore((s) => s.undoMesh) const redoMesh = useAppStore((s) => s.redoMesh) const canUndo = useAppStore((s) => s.historyIndex > 0) const canRedo = useAppStore((s) => s.historyIndex < s.meshHistory.length - 1) const { optimizeMesh, smoothMesh, importMesh } = useApi() + const assetLibraryService = useMemo(() => getDefaultAssetLibraryService(), []) useEffect(() => { const handler = (e: KeyboardEvent) => { if (!e.ctrlKey && !e.metaKey) return - if (e.key === 'z') { e.preventDefault(); undoMesh() } + if (e.key === 'z') { e.preventDefault(); if (gizmoUndoRef.current?.()) return; undoMesh() } if (e.key === 'y') { e.preventDefault(); redoMesh() } } window.addEventListener('keydown', handler) @@ -307,6 +612,33 @@ export default function GeneratePage(): JSX.Element { const hasModel = currentJob?.status === 'done' && !!currentJob.outputUrl + // Drop the active transform tool when the mesh is deselected, so it doesn't + // silently re-activate on the next selection. + useEffect(() => { + if (!meshSelected) setGizmoMode(null) + }, [meshSelected]) + + // Gizmo hotkeys: W move, R rotate, S scale, Esc exits. Ignored while typing. + useEffect(() => { + const handler = (e: KeyboardEvent) => { + const el = document.activeElement as HTMLElement | null + if (el && (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el.isContentEditable)) return + if (e.key === 'Escape') { setGizmoMode((m) => (m ? null : m)); return } + if (!hasModel || !meshSelected) return + const k = e.key.toLowerCase() + if (k === 'w') setGizmoMode('translate') + else if (k === 'r') setGizmoMode('rotate') + else if (k === 's') setGizmoMode('scale') + } + window.addEventListener('keydown', handler) + return () => window.removeEventListener('keydown', handler) + }, [hasModel, meshSelected]) + + useEffect(() => { + if (openPanel !== 'library' || libraryLoaded || libraryLoading) return + void loadLibraryEntries() + }, [openPanel, libraryLoaded, libraryLoading]) + async function handleUnloadAll() { await window.electron.model.unloadAll() setUnloadStatus('done') @@ -327,6 +659,16 @@ export default function GeneratePage(): JSX.Element { link.click() } + function getOptimizePath(url: string): string { + if (url.startsWith('/workspace/')) { + return url.slice('/workspace/'.length) + } + if (url.startsWith('/optimize/serve-file?path=')) { + return decodeURIComponent(url.split('path=')[1] ?? '') + } + return url + } + async function handleImportMesh() { const filePath = await window.electron.fs.selectMeshFile() if (!filePath) return @@ -350,15 +692,81 @@ export default function GeneratePage(): JSX.Element { } } + async function loadLibraryEntries() { + setLibraryLoading(true) + setLibraryError(null) + try { + const result = await assetLibraryService.list() + if (!result.success) { + setLibraryLoaded(false) + setLibraryEntries([]) + setLibrarySelectedEntryId(null) + setLibraryError(result.error.message) + return + } + setLibraryEntries(result.entries) + setLibrarySelectedEntryId((current) => current && result.entries.some((entry) => entry.id === current) + ? current + : result.entries.find(isAssetLibraryEntryOpenable)?.id ?? result.entries[0]?.id ?? null) + setLibraryLoaded(true) + } catch (err) { + setLibraryLoaded(false) + setLibraryEntries([]) + setLibrarySelectedEntryId(null) + setLibraryError(err instanceof Error ? err.message : String(err)) + } finally { + setLibraryLoading(false) + } + } + + async function handleOpenSelectedLibraryEntry() { + const selectedEntry = libraryEntries.find((entry) => entry.id === librarySelectedEntryId) ?? null + if (!selectedEntry) { + setLibraryError('Select an asset before opening it in Generate.') + return + } + if (!isAssetLibraryEntryOpenable(selectedEntry)) { + setLibraryError(describeAssetLibraryOpenability(selectedEntry)) + return + } + + setLibraryOpening(true) + setLibraryError(null) + try { + const result = await assetLibraryService.open(buildAssetLibraryOpenRequest(selectedEntry)) + if (!result.success) { + setLibraryError(result.error.message) + return + } + const target = resolveAssetLibraryOpenTarget(result.entry) + const selection = createAssetLibraryOpenJob(result.entry, target) + if (!selection) { + setLibraryError(describeAssetLibraryOpenability(result.entry)) + return + } + setLibraryEntries((currentEntries) => currentEntries.map((entry) => entry.id === result.entry.id ? result.entry : entry)) + setLibrarySelectedEntryId(result.entry.id) + setCurrentJob(selection.job) + pushMeshUrl(selection.historyUrl) + setOpenPanel((currentPanel) => resolveOpenPanelAfterLibrarySelection(currentPanel)) + } catch (err) { + setLibraryError(err instanceof Error ? err.message : String(err)) + } finally { + setLibraryOpening(false) + } + } + async function handleSmooth(iterations: number) { if (!currentJob?.outputUrl) return setSmoothing(true) try { - const path = currentJob.outputUrl.replace('/workspace/', '') + const path = getOptimizePath(currentJob.outputUrl) const { url } = await smoothMesh(path, iterations) updateCurrentJob({ outputUrl: url }) pushMeshUrl(url) setOpenPanel(null) + } catch (err) { + showError(err instanceof Error ? err.message : String(err)) } finally { setSmoothing(false) } @@ -368,11 +776,13 @@ export default function GeneratePage(): JSX.Element { if (!currentJob?.outputUrl) return setDecimating(true) try { - const path = currentJob.outputUrl.replace('/workspace/', '') + const path = getOptimizePath(currentJob.outputUrl) const { url } = await optimizeMesh(path, targetFaces) updateCurrentJob({ outputUrl: url }) pushMeshUrl(url) setOpenPanel(null) + } catch (err) { + showError(err instanceof Error ? err.message : String(err)) } finally { setDecimating(false) } @@ -411,6 +821,21 @@ export default function GeneratePage(): JSX.Element { {/* Header bar */}
+ {/* Free memory */} + + +
+ {/* Undo / Redo */}
)}
+
+ { + setLibraryError(null) + setOpenPanel((panel) => (panel === 'library' ? null : 'library')) + }} + /> + {openPanel === 'library' && ( + { + setLibraryError(null) + setLibrarySelectedEntryId(entryId) + }} + onSearchQueryChange={setLibrarySearchQuery} + onSortModeChange={setLibrarySortMode} + onToggleSection={(sectionKey) => setLibraryCollapsedSectionKeys((current) => toggleAssetLibrarySectionKey(current, sectionKey))} + onOpenSelected={() => { void handleOpenSelectedLibraryEntry() }} + onRefresh={() => { void loadLibraryEntries() }} + onClose={() => setOpenPanel(null)} + /> + )} +
+ {hasModel && ( <>
@@ -619,25 +1077,56 @@ export default function GeneratePage(): JSX.Element {
+ {/* Tools bar β€” always visible; transform tools appear once a mesh is selected */} +
+ {hasModel && meshSelected && ( + <> + setGizmoMode((m) => (m === 'translate' ? null : 'translate'))} + > + + + + + + + + + + setGizmoMode((m) => (m === 'rotate' ? null : 'rotate'))} + > + + + + + + setGizmoMode((m) => (m === 'scale' ? null : 'scale'))} + > + + + + + + + + + )} +
+ {/* Viewer area */}
- + - - {/* Free memory β€” overlay top-left */} -
) -} +} \ No newline at end of file diff --git a/src/areas/generate/assetLibraryProjection.test.ts b/src/areas/generate/assetLibraryProjection.test.ts new file mode 100644 index 0000000..bc1c9df --- /dev/null +++ b/src/areas/generate/assetLibraryProjection.test.ts @@ -0,0 +1,112 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + filterAssetLibraryEntries, + groupAssetLibraryEntries, + projectAssetLibraryEntry, + resolveAssetLibraryOpenTarget, +} from './assetLibraryProjection.ts' +import type { AssetLibraryEntry } from '../../shared/types/assetLibrary.ts' + +const glbEntry: AssetLibraryEntry = { + id: 'library:Workflows/checkpoints/hero.glb', + workspacePath: 'Workflows/checkpoints/hero.glb', + displayName: 'hero.glb', + sourceScope: 'workflows', + capability: 'mesh', + state: 'ready', + previewKind: '3d-model', + warnings: ['safe', 'safe'], + openable: true, +} + +test('projects entries with deduped warnings and workspace URL open target', () => { + const projected = projectAssetLibraryEntry(glbEntry) + assert.deepEqual(projected.warnings, ['safe']) + assert.deepEqual(resolveAssetLibraryOpenTarget(projected), { + kind: 'self', + url: '/workspace/Workflows/checkpoints/hero.glb', + workspacePath: 'Workflows/checkpoints/hero.glb', + }) +}) + +test('projects sidecars with safe GLB source links as linked-source open targets', () => { + const projected = projectAssetLibraryEntry({ + ...glbEntry, + id: 'library:Workflows/checkpoints/hero.landmarks.v1.json', + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + displayName: 'hero.landmarks.v1.json', + capability: 'landmarks-sidecar', + previewKind: 'text', + openable: false, + nonOpenableReason: 'Open the linked source mesh.', + source: { workspacePath: 'Workflows/checkpoints/hero.glb', displayName: 'hero.glb', role: 'source-mesh' }, + manifest: { workspacePath: 'Workflows/checkpoints/hero.scene.json', capability: 'scene-manifest' }, + }) + + assert.deepEqual(projected.source, { workspacePath: 'Workflows/checkpoints/hero.glb', displayName: 'hero.glb', role: 'source-mesh' }) + assert.deepEqual(projected.manifest, { workspacePath: 'Workflows/checkpoints/hero.scene.json', capability: 'scene-manifest' }) + assert.deepEqual(resolveAssetLibraryOpenTarget(projected), { + kind: 'linked-source', + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + sourceWorkspacePath: 'Workflows/checkpoints/hero.glb', + url: '/workspace/Workflows/checkpoints/hero.glb', + }) +}) + +test('degrades unsafe source and manifest paths and unsupported formats to unavailable targets', () => { + const projected = projectAssetLibraryEntry({ + ...glbEntry, + id: 'unsafe-source', + workspacePath: 'Workflows/checkpoints/hero.landmarks.v1.json', + displayName: 'hero.landmarks.v1.json', + capability: 'landmarks-sidecar', + previewKind: 'text', + openable: false, + source: { workspacePath: 'Workflows/%2e%2e/secret.glb' }, + manifest: { workspacePath: '../secret.scene.json', capability: 'scene-manifest' }, + }) + const splat = projectAssetLibraryEntry({ ...glbEntry, id: 'splat', workspacePath: 'Workflows/scan.splat', displayName: 'scan.splat', openable: true }) + + assert.equal(projected.source, undefined) + assert.equal(projected.manifest, undefined) + assert.deepEqual(projected.warnings, ['safe', 'Ignored unsafe source workspace path.', 'Ignored unsafe manifest workspace path.']) + assert.deepEqual(resolveAssetLibraryOpenTarget(projected), { + kind: 'unavailable', + reason: 'Workspace asset is not openable.', + }) + assert.deepEqual(resolveAssetLibraryOpenTarget(splat), { + kind: 'unavailable', + reason: 'Only safe .glb/.gltf workspace assets are openable in this release.', + }) +}) + +test('marks unsafe or non-openable entries unavailable without throwing', () => { + const projected = projectAssetLibraryEntry({ + ...glbEntry, + workspacePath: '../escape.glb', + state: 'unsafe', + openable: false, + nonOpenableReason: 'Unsafe workspace path.', + }) + assert.equal(projected.state, 'unsafe') + assert.deepEqual(resolveAssetLibraryOpenTarget(projected), { + kind: 'unavailable', + reason: 'Unsafe workspace path.', + }) +}) + +test('filters by name, path, capability, scope, and groups by scope then capability', () => { + const entries = [ + projectAssetLibraryEntry(glbEntry), + projectAssetLibraryEntry({ ...glbEntry, id: 'export', workspacePath: 'Exports/static.ply', displayName: 'static.ply', sourceScope: 'exports', openable: false }), + ] + assert.deepEqual(filterAssetLibraryEntries(entries, 'exports').map((entry) => entry.workspacePath), ['Exports/static.ply']) + assert.deepEqual(filterAssetLibraryEntries(entries, 'hero.glb').map((entry) => entry.displayName), ['hero.glb']) + assert.deepEqual(filterAssetLibraryEntries(entries, 'mesh').map((entry) => entry.displayName), ['hero.glb', 'static.ply']) + assert.deepEqual(groupAssetLibraryEntries(entries).map((group) => [group.scope, group.capabilityGroups.map((capability) => capability.capability)]), [ + ['exports', ['mesh']], + ['workflows', ['mesh']], + ]) +}) diff --git a/src/areas/generate/assetLibraryProjection.ts b/src/areas/generate/assetLibraryProjection.ts new file mode 100644 index 0000000..b65f5b5 --- /dev/null +++ b/src/areas/generate/assetLibraryProjection.ts @@ -0,0 +1,99 @@ +import type { AssetCapability, AssetLibraryEntry, AssetLibrarySourceScope } from '../../shared/types/assetLibrary' + +export interface ProjectedAssetLibraryEntry extends AssetLibraryEntry { + warnings: string[] +} + +export type AssetLibraryOpenTarget = + | { kind: 'self', url: string, workspacePath: string } + | { kind: 'linked-source', url: string, workspacePath: string, sourceWorkspacePath: string } + | { kind: 'unavailable', reason: string } + +export interface AssetLibraryCapabilityGroup { + capability: AssetCapability | 'uncategorized' + entries: ProjectedAssetLibraryEntry[] +} + +export interface AssetLibraryScopeGroup { + scope: AssetLibrarySourceScope + capabilityGroups: AssetLibraryCapabilityGroup[] +} + +function isSafeWorkspacePath(workspacePath: string): boolean { + const normalized = workspacePath.replace(/\\/g, '/').trim() + return /^(Workflows|Exports)\//.test(normalized) + && !normalized.split('/').includes('..') + && !/%2e|%2f|%5c/i.test(normalized) + && !normalized.startsWith('/') + && !/^[a-zA-Z]:[\\/]/.test(workspacePath) + && !workspacePath.startsWith('\\\\') +} + +function isGlbOrGltf(workspacePath: string): boolean { + return /\.(glb|gltf)$/i.test(workspacePath) +} + +export function projectAssetLibraryEntry(entry: AssetLibraryEntry): ProjectedAssetLibraryEntry { + const warnings = [...new Set(entry.warnings)] + if (!isSafeWorkspacePath(entry.workspacePath)) { + return { ...entry, state: 'unsafe', openable: false, nonOpenableReason: entry.nonOpenableReason ?? 'Unsafe workspace path.', warnings } + } + const safeSource = entry.source && isSafeWorkspacePath(entry.source.workspacePath) ? entry.source : undefined + const safeManifest = entry.manifest && isSafeWorkspacePath(entry.manifest.workspacePath) ? entry.manifest : undefined + if (entry.source && !safeSource) warnings.push('Ignored unsafe source workspace path.') + if (entry.manifest && !safeManifest) warnings.push('Ignored unsafe manifest workspace path.') + return { ...entry, source: safeSource, manifest: safeManifest, warnings: [...new Set(warnings)] } +} + +export function resolveAssetLibraryOpenTarget(entry: ProjectedAssetLibraryEntry): AssetLibraryOpenTarget { + if (entry.state !== 'ready') { + return { kind: 'unavailable', reason: entry.nonOpenableReason ?? 'Workspace asset is not openable.' } + } + if (entry.source?.workspacePath) { + if (!isSafeWorkspacePath(entry.source.workspacePath) || entry.source.workspacePath === entry.workspacePath || !isGlbOrGltf(entry.source.workspacePath)) { + return { kind: 'unavailable', reason: 'Linked source mesh is unavailable.' } + } + return { + kind: 'linked-source', + url: `/workspace/${entry.source.workspacePath}`, + workspacePath: entry.workspacePath, + sourceWorkspacePath: entry.source.workspacePath, + } + } + if (!entry.openable) { + return { kind: 'unavailable', reason: entry.nonOpenableReason ?? 'Workspace asset is not openable.' } + } + if (!isGlbOrGltf(entry.workspacePath)) { + return { kind: 'unavailable', reason: 'Only safe .glb/.gltf workspace assets are openable in this release.' } + } + return { kind: 'self', url: `/workspace/${entry.workspacePath}`, workspacePath: entry.workspacePath } +} + +export function filterAssetLibraryEntries(entries: ProjectedAssetLibraryEntry[], query: string): ProjectedAssetLibraryEntry[] { + const needle = query.trim().toLowerCase() + if (!needle) return entries + return entries.filter((entry) => [ + entry.displayName, + entry.workspacePath, + entry.capability ?? '', + entry.sourceScope, + entry.state, + entry.source?.workspacePath ?? '', + entry.manifest?.workspacePath ?? '', + ].some((value) => value.toLowerCase().includes(needle))) +} + +export function groupAssetLibraryEntries(entries: ProjectedAssetLibraryEntry[]): AssetLibraryScopeGroup[] { + const scopes = [...new Set(entries.map((entry) => entry.sourceScope))].sort() + return scopes.map((scope) => { + const scopeEntries = entries.filter((entry) => entry.sourceScope === scope) + const capabilities = [...new Set(scopeEntries.map((entry) => entry.capability ?? 'uncategorized'))].sort() + return { + scope, + capabilityGroups: capabilities.map((capability) => ({ + capability, + entries: scopeEntries.filter((entry) => (entry.capability ?? 'uncategorized') === capability), + })), + } + }) +} diff --git a/src/areas/generate/assetLibraryService.test.ts b/src/areas/generate/assetLibraryService.test.ts new file mode 100644 index 0000000..9e7938c --- /dev/null +++ b/src/areas/generate/assetLibraryService.test.ts @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { createAssetLibraryService } from './assetLibraryService.ts' + +test('renderer service validates requests before IPC and projects structured errors', async () => { + let invoked = false + const service = createAssetLibraryService({ + list: async () => ({ success: true, entries: [] }), + read: async () => { invoked = true; return { success: false, error: { code: 'unexpected', message: 'should not run' } } }, + open: async () => { invoked = true; return { success: false, error: { code: 'unexpected', message: 'should not run' } } }, + }) + + const read = await service.read({ workspacePath: '../escape.glb' }) + const open = await service.open({ workspacePath: '/tmp/hero.glb' }) + assert.equal(read.success, false) + assert.equal(!read.success && read.error.code, 'unsafe-path') + assert.equal(open.success, false) + assert.equal(!open.success && open.error.code, 'unsafe-path') + assert.equal(invoked, false) +}) + +test('renderer service validates sourceWorkspacePath before read and open IPC calls', async () => { + let invoked = false + const service = createAssetLibraryService({ + list: async () => ({ success: true, entries: [] }), + read: async () => { invoked = true; return { success: false, error: { code: 'unexpected', message: 'should not run' } } }, + open: async () => { invoked = true; return { success: false, error: { code: 'unexpected', message: 'should not run' } } }, + }) + + const read = await service.read({ workspacePath: 'Workflows/hero.landmarks.v1.json', sourceWorkspacePath: '../secret.glb' }) + const open = await service.open({ workspacePath: 'Workflows/hero.landmarks.v1.json', sourceWorkspacePath: 'C:\\secret.glb' }) + + assert.equal(read.success, false) + assert.equal(!read.success && read.error.code, 'unsafe-path') + assert.equal(open.success, false) + assert.equal(!open.success && open.error.code, 'unsafe-path') + assert.equal(invoked, false) +}) + +test('renderer service delegates safe IPC calls and normalizes returned entries', async () => { + let openRequest: unknown = null + const service = createAssetLibraryService({ + list: async () => ({ + success: true, + entries: [{ + id: 'asset', workspacePath: 'Workflows/hero.glb', displayName: 'hero.glb', sourceScope: 'workflows', + capability: 'mesh', state: 'ready', previewKind: '3d-model', warnings: ['a', 'a'], openable: true, + }], + }), + read: async () => ({ success: false, error: { code: 'missing', message: 'Missing' } }), + open: async (request) => { openRequest = request; return { success: false, error: { code: 'missing', message: 'Missing' } } }, + }) + + const result = await service.list() + assert.equal(result.success, true) + assert.deepEqual(result.success && result.entries[0]?.warnings, ['a']) + await service.open({ workspacePath: 'Workflows/hero.landmarks.v1.json', sourceWorkspacePath: 'Workflows/hero.glb' }) + assert.deepEqual(openRequest, { workspacePath: 'Workflows/hero.landmarks.v1.json', sourceWorkspacePath: 'Workflows/hero.glb' }) +}) diff --git a/src/areas/generate/assetLibraryService.ts b/src/areas/generate/assetLibraryService.ts new file mode 100644 index 0000000..3873857 --- /dev/null +++ b/src/areas/generate/assetLibraryService.ts @@ -0,0 +1,68 @@ +import type { + AssetLibraryError, + AssetLibraryListResult, + AssetLibraryOpenRequest, + AssetLibraryOpenResult, + AssetLibraryReadRequest, + AssetLibraryReadResult, +} from '../../shared/types/assetLibrary' +import { projectAssetLibraryEntry, type ProjectedAssetLibraryEntry } from './assetLibraryProjection' + +export interface AssetLibraryPreloadApi { + list: () => Promise + read: (request: AssetLibraryReadRequest) => Promise + open: (request: AssetLibraryOpenRequest) => Promise +} + +export type ProjectedAssetLibraryListResult = + | { success: true, entries: ProjectedAssetLibraryEntry[] } + | { success: false, error: AssetLibraryError } + +export type ProjectedAssetLibraryReadResult = + | { success: true, entry: ProjectedAssetLibraryEntry, preview: Extract['preview'] } + | { success: false, error: AssetLibraryError } + +export type ProjectedAssetLibraryOpenResult = + | { success: true, entry: ProjectedAssetLibraryEntry } + | { success: false, error: AssetLibraryError } + +function unsafeError(message = 'Workspace path must stay under Workflows/ or Exports/.'): AssetLibraryError { + return { code: 'unsafe-path', message } +} + +function validateWorkspacePath(workspacePath: string): AssetLibraryError | null { + const trimmed = workspacePath.trim() + if (!trimmed || trimmed.startsWith('/') || /^[a-zA-Z]:[\\/]/.test(trimmed) || trimmed.startsWith('\\\\')) return unsafeError() + if (/%2e|%2f|%5c/i.test(trimmed) || trimmed.split(/[\\/]+/).includes('..')) return unsafeError() + const normalized = trimmed.replace(/\\/g, '/') + if (!normalized.startsWith('Workflows/') && !normalized.startsWith('Exports/')) return unsafeError() + return null +} + +export function createAssetLibraryService(api: AssetLibraryPreloadApi) { + return { + async list(): Promise { + const result = await api.list() + if (!result.success) return result + return { success: true, entries: result.entries.map(projectAssetLibraryEntry) } + }, + async read(request: AssetLibraryReadRequest): Promise { + const invalid = validateWorkspacePath(request.workspacePath) ?? (request.sourceWorkspacePath ? validateWorkspacePath(request.sourceWorkspacePath) : null) + if (invalid) return { success: false, error: invalid } + const result = await api.read(request) + if (!result.success) return result + return { success: true, entry: projectAssetLibraryEntry(result.entry), preview: result.preview } + }, + async open(request: AssetLibraryOpenRequest): Promise { + const invalid = validateWorkspacePath(request.workspacePath) ?? (request.sourceWorkspacePath ? validateWorkspacePath(request.sourceWorkspacePath) : null) + if (invalid) return { success: false, error: invalid } + const result = await api.open(request) + if (!result.success) return result + return { success: true, entry: projectAssetLibraryEntry(result.entry) } + }, + } +} + +export function getDefaultAssetLibraryService() { + return createAssetLibraryService(window.electron.workspace.library) +} diff --git a/src/areas/generate/assetLibraryUi.test.ts b/src/areas/generate/assetLibraryUi.test.ts new file mode 100644 index 0000000..e6b9fbc --- /dev/null +++ b/src/areas/generate/assetLibraryUi.test.ts @@ -0,0 +1,125 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + buildAssetLibraryOpenRequest, + createAssetLibraryOpenJob, + describeAssetLibraryOpenability, + filterAssetLibraryScopeGroups, + getDefaultAssetLibraryCollapsedSectionKeys, + isAssetLibraryEntryOpenable, + resolveOpenPanelAfterLibrarySelection, + toggleAssetLibrarySectionKey, + type AssetLibrarySortMode, + type GenerateOpenPanel, +} from './assetLibraryUi.ts' +import { projectAssetLibraryEntry, resolveAssetLibraryOpenTarget, type ProjectedAssetLibraryEntry } from './assetLibraryProjection.ts' +import type { AssetLibraryEntry } from '../../shared/types/assetLibrary.ts' + +function entry(overrides: Partial = {}): ProjectedAssetLibraryEntry { + const base: AssetLibraryEntry = { + id: 'library:Workflows/hero.glb', + workspacePath: 'Workflows/hero.glb', + displayName: 'hero.glb', + sourceScope: 'workflows', + capability: 'mesh', + state: 'ready', + previewKind: '3d-model', + warnings: [], + openable: true, + createdAt: '2026-06-16T10:00:00.000Z', + ...overrides, + } + return projectAssetLibraryEntry(base) +} + +function groupPaths(entries: ProjectedAssetLibraryEntry[], search = '', sortMode: AssetLibrarySortMode = 'type'): string[] { + return filterAssetLibraryScopeGroups(entries, search, sortMode).flatMap((scopeGroup) => ( + scopeGroup.entryGroups.flatMap((group) => group.entries.map((item) => item.workspacePath)) + )) +} + +test('organizes visible library assets by scope and capability with collapsible section keys', () => { + const entries = [ + entry({ id: 'workflow-mesh', workspacePath: 'Workflows/run/hero.glb', displayName: 'hero.glb', sourceScope: 'workflows', capability: 'mesh' }), + entry({ id: 'export-rig', workspacePath: 'Exports/rig/hero-rig.gltf', displayName: 'hero-rig.gltf', sourceScope: 'exports', capability: 'rigged-mesh' }), + entry({ id: 'hidden-cache', workspacePath: 'Workflows/run/cache/internal.glb', displayName: 'internal.glb', sourceScope: 'workflows' }), + entry({ id: 'unsupported', workspacePath: 'Exports/readme.txt', displayName: 'readme.txt', sourceScope: 'exports', state: 'unsupported', openable: false }), + ] + + const groups = filterAssetLibraryScopeGroups(entries, '', 'type') + assert.deepEqual(groups.map((group) => [group.sourceScope, group.entryGroups.map((entryGroup) => entryGroup.capability)]), [ + ['workflows', ['mesh']], + ['exports', ['rigged-mesh']], + ]) + assert.deepEqual(groupPaths(entries), ['Workflows/run/hero.glb', 'Exports/rig/hero-rig.gltf']) + assert.equal(getDefaultAssetLibraryCollapsedSectionKeys().includes('scope:workflows'), true) + assert.deepEqual(toggleAssetLibrarySectionKey(['scope:workflows'], 'scope:workflows'), []) + assert.deepEqual(toggleAssetLibrarySectionKey([], 'scope:exports'), ['scope:exports']) +}) + +test('searches workspace assets by name path capability scope source and manifest while supporting name/date sorting', () => { + const entries = [ + entry({ id: 'b', workspacePath: 'Workflows/run/zebra.glb', displayName: 'zebra.glb', sourceScope: 'workflows', capability: 'mesh', createdAt: '2026-06-15T10:00:00.000Z' }), + entry({ id: 'a', workspacePath: 'Exports/rig/alpha.gltf', displayName: 'alpha.gltf', sourceScope: 'exports', capability: 'rigged-mesh', createdAt: '2026-06-16T10:00:00.000Z' }), + entry({ + id: 'c', workspacePath: 'Exports/motion/walk.json', displayName: 'walk.json', sourceScope: 'exports', capability: 'animation-motion', openable: false, previewKind: 'text', + source: { workspacePath: 'Workflows/run/zebra.glb', displayName: 'zebra.glb' }, + manifest: { workspacePath: 'Exports/motion/walk.scene.json', capability: 'scene-manifest' }, + }), + ] + + assert.deepEqual(groupPaths(entries, 'rigged'), ['Exports/rig/alpha.gltf']) + assert.deepEqual(groupPaths(entries, 'walk.scene'), ['Exports/motion/walk.json']) + assert.deepEqual(groupPaths(entries, 'zebra.glb'), ['Workflows/run/zebra.glb', 'Exports/motion/walk.json']) + assert.deepEqual(groupPaths(entries, 'exports', 'name'), ['Exports/rig/alpha.gltf', 'Exports/motion/walk.json']) + assert.deepEqual(groupPaths(entries, '', 'date'), ['Workflows/run/zebra.glb', 'Exports/rig/alpha.gltf', 'Exports/motion/walk.json']) +}) + +test('opens only safe glb and gltf entries through existing Generate job and history state', () => { + const glb = entry({ workspacePath: 'Workflows/run/hero.glb', displayName: 'hero.glb' }) + const ply = entry({ workspacePath: 'Exports/scan.ply', displayName: 'scan.ply', openable: false, nonOpenableReason: 'Only .glb/.gltf workspace assets are openable in this release.' }) + + assert.equal(isAssetLibraryEntryOpenable(glb), true) + assert.equal(isAssetLibraryEntryOpenable(ply), false) + assert.equal(describeAssetLibraryOpenability(glb), 'Ready to open this asset directly in Generate.') + assert.equal(describeAssetLibraryOpenability(ply), 'Only .glb/.gltf workspace assets are openable in this release.') + assert.deepEqual(buildAssetLibraryOpenRequest(glb), { workspacePath: 'Workflows/run/hero.glb' }) + + const target = resolveAssetLibraryOpenTarget(glb) + assert.equal(target.kind, 'self') + if (target.kind !== 'self') throw new Error('expected self target') + + const selection = createAssetLibraryOpenJob(glb, target, 1718546400000) + assert.equal(selection.historyUrl, '/workspace/Workflows/run/hero.glb') + assert.equal(selection.job.status, 'done') + assert.equal(selection.job.outputUrl, '/workspace/Workflows/run/hero.glb') + assert.equal(selection.job.originalOutputUrl, '/workspace/Workflows/run/hero.glb') + assert.equal(resolveOpenPanelAfterLibrarySelection('library' satisfies GenerateOpenPanel), 'library') +}) + +test('builds linked-source open requests and import jobs for safe sidecars', () => { + const sidecar = entry({ + id: 'sidecar', + workspacePath: 'Workflows/run/hero.landmarks.v1.json', + displayName: 'hero.landmarks.v1.json', + capability: 'landmarks-sidecar', + previewKind: 'text', + openable: false, + source: { workspacePath: 'Workflows/run/hero.glb', displayName: 'hero.glb' }, + }) + + assert.equal(isAssetLibraryEntryOpenable(sidecar), true) + assert.equal(describeAssetLibraryOpenability(sidecar), 'Ready to open linked source hero.glb in Generate.') + assert.deepEqual(buildAssetLibraryOpenRequest(sidecar), { + workspacePath: 'Workflows/run/hero.landmarks.v1.json', + sourceWorkspacePath: 'Workflows/run/hero.glb', + }) + + const target = resolveAssetLibraryOpenTarget(sidecar) + assert.equal(target.kind, 'linked-source') + if (target.kind !== 'linked-source') throw new Error('expected linked source target') + const selection = createAssetLibraryOpenJob(sidecar, target, 1718546400001) + assert.equal(selection?.historyUrl, '/workspace/Workflows/run/hero.glb') + assert.equal(selection?.job.outputUrl, '/workspace/Workflows/run/hero.glb') +}) diff --git a/src/areas/generate/assetLibraryUi.ts b/src/areas/generate/assetLibraryUi.ts new file mode 100644 index 0000000..fb7e581 --- /dev/null +++ b/src/areas/generate/assetLibraryUi.ts @@ -0,0 +1,252 @@ +import type { GenerationJob } from '../../shared/stores/appStore' +import type { AssetLibraryOpenRequest } from '../../shared/types/assetLibrary' +import { resolveAssetLibraryOpenTarget, type AssetLibraryOpenTarget, type ProjectedAssetLibraryEntry } from './assetLibraryProjection' + +export type GenerateOpenPanel = 'export' | 'decimate' | 'smooth' | 'import' | 'library' | 'light' | null +export type AssetLibrarySortMode = 'type' | 'name' | 'date' + +export interface AssetLibraryEntryGroup { + capability: NonNullable + capabilityLabel: string + sectionKey: string + entries: ProjectedAssetLibraryEntry[] +} + +export interface AssetLibrarySourceScopeGroup { + sourceScope: ProjectedAssetLibraryEntry['sourceScope'] + sourceScopeLabel: string + sectionKey: string + entryGroups: AssetLibraryEntryGroup[] +} + +export interface AssetLibraryOpenSelection { + historyUrl: string + job: GenerationJob +} + +const WORKSPACE_URL_PREFIX = '/workspace/' + +const ASSET_LIBRARY_CAPABILITY_SECTIONS = [ + { capability: 'mesh', label: 'Mesh' }, + { capability: 'rigged-mesh', label: 'Rigged mesh' }, + { capability: 'animation-motion', label: 'Animations/motions' }, + { capability: 'landmarks-sidecar', label: 'Landmarks sidecars' }, + { capability: 'generated-world', label: 'Generated worlds' }, + { capability: 'scene-manifest', label: 'Scene manifests' }, +] as const satisfies ReadonlyArray<{ capability: NonNullable, label: string }> + +const ASSET_LIBRARY_SOURCE_SCOPE_SECTIONS = [ + { sourceScope: 'workflows', label: 'Workflows' }, + { sourceScope: 'exports', label: 'Exports' }, +] as const satisfies ReadonlyArray<{ sourceScope: ProjectedAssetLibraryEntry['sourceScope'], label: string }> + +const ASSET_LIBRARY_CAPABILITY_ORDER = new Map( + ASSET_LIBRARY_CAPABILITY_SECTIONS.map((section, index) => [section.capability, index]), +) + +const ASSET_LIBRARY_INTERNAL_DIRECTORY_NAMES = new Set(['tmp', 'temp', 'cache']) + +export const ASSET_LIBRARY_SORT_OPTIONS = [ + { value: 'type', label: 'Type' }, + { value: 'name', label: 'Name' }, + { value: 'date', label: 'Date' }, +] as const satisfies ReadonlyArray<{ value: AssetLibrarySortMode, label: string }> + +export function getDefaultAssetLibraryCollapsedSectionKeys(): string[] { + const sectionKeys = ASSET_LIBRARY_SOURCE_SCOPE_SECTIONS.flatMap((scopeSection) => { + const capabilityKeys = ASSET_LIBRARY_CAPABILITY_SECTIONS.map( + (capabilitySection) => `capability:${scopeSection.sourceScope}:${capabilitySection.capability}`, + ) + + return [`scope:${scopeSection.sourceScope}`, ...capabilityKeys] + }) + + return [...sectionKeys] +} + +export function toggleAssetLibrarySectionKey(currentKeys: string[], sectionKey: string): string[] { + return currentKeys.includes(sectionKey) + ? currentKeys.filter((value) => value !== sectionKey) + : [...currentKeys, sectionKey] +} + +export function buildAssetLibraryOpenRequest(entry: ProjectedAssetLibraryEntry): AssetLibraryOpenRequest { + const target = resolveAssetLibraryOpenTarget(entry) + return target.kind === 'linked-source' + ? { workspacePath: target.workspacePath, sourceWorkspacePath: target.sourceWorkspacePath } + : { workspacePath: entry.workspacePath } +} + +export function isAssetLibraryEntryOpenable(entry: ProjectedAssetLibraryEntry | null | undefined): entry is ProjectedAssetLibraryEntry { + return Boolean(entry && resolveAssetLibraryOpenTarget(entry).kind !== 'unavailable') +} + +export function describeAssetLibraryOpenability(entry: ProjectedAssetLibraryEntry): string { + if (entry.state === 'unknown-metadata') return 'Missing metadata prevents a safe open in Generate.' + if (entry.state === 'unsupported') return 'This asset is tracked in the library but is not supported in Generate.' + if (entry.state === 'unsafe') return 'This asset was rejected because its workspace path is unsafe.' + const target = resolveAssetLibraryOpenTarget(entry) + if (target.kind === 'linked-source') return `Ready to open linked source ${entry.source?.displayName ?? target.sourceWorkspacePath} in Generate.` + if (target.kind === 'self') return 'Ready to open this asset directly in Generate.' + if (entry.nonOpenableReason) return entry.nonOpenableReason + if (!/\.(glb|gltf)$/i.test(entry.workspacePath)) return 'Only .glb/.gltf workspace assets are openable in this release.' + return target.reason +} + +export function filterVisibleAssetLibraryEntries(entries: ProjectedAssetLibraryEntry[]): ProjectedAssetLibraryEntry[] { + return entries.filter((entry) => entry.state !== 'unsupported' && !hasInternalAssetLibraryDirectory(entry.workspacePath)) +} + +export function filterAssetLibraryScopeGroups( + entries: ProjectedAssetLibraryEntry[], + searchQuery: string, + sortMode: AssetLibrarySortMode, +): AssetLibrarySourceScopeGroup[] { + const normalizedSearchQuery = normalizeAssetLibrarySearchQuery(searchQuery) + const visibleEntries = filterVisibleAssetLibraryEntries(entries) + + return ASSET_LIBRARY_SOURCE_SCOPE_SECTIONS + .map((scopeSection) => { + const scopeEntries = visibleEntries.filter((entry) => entry.sourceScope === scopeSection.sourceScope) + const scopeMatches = normalizedSearchQuery.length > 0 && matchesAssetLibrarySearch(scopeSection.label, normalizedSearchQuery) + + const entryGroups = ASSET_LIBRARY_CAPABILITY_SECTIONS + .map((capabilitySection) => { + const capabilityEntries = scopeEntries.filter((entry) => entry.capability === capabilitySection.capability) + if (capabilityEntries.length === 0) return null + + const capabilityMatches = scopeMatches || (normalizedSearchQuery.length > 0 && matchesAssetLibrarySearch(capabilitySection.label, normalizedSearchQuery)) + const visibleCapabilityEntries = !normalizedSearchQuery || capabilityMatches + ? capabilityEntries + : capabilityEntries.filter((entry) => matchesAssetLibraryEntrySearch(entry, normalizedSearchQuery)) + + if (visibleCapabilityEntries.length === 0) return null + + return { + capability: capabilitySection.capability, + capabilityLabel: capabilitySection.label, + sectionKey: `capability:${scopeSection.sourceScope}:${capabilitySection.capability}`, + entries: sortAssetLibraryEntries(visibleCapabilityEntries, sortMode), + } + }) + .filter((group): group is AssetLibraryEntryGroup => group !== null) + + const sortedEntryGroups = sortMode === 'type' + ? entryGroups + : [...entryGroups].sort((left, right) => compareAssetLibraryEntryGroups(left, right, sortMode)) + + if (sortedEntryGroups.length === 0) return null + return { + sourceScope: scopeSection.sourceScope, + sourceScopeLabel: scopeSection.label, + sectionKey: `scope:${scopeSection.sourceScope}`, + entryGroups: sortedEntryGroups, + } + }) + .filter((group): group is AssetLibrarySourceScopeGroup => group !== null) +} + +export function createAssetLibraryOpenJob( + entry: ProjectedAssetLibraryEntry, + target: AssetLibraryOpenTarget, + now = Date.now(), +): AssetLibraryOpenSelection | null { + if (target.kind === 'unavailable') return null + return { + historyUrl: target.url, + job: { + id: `library-${now}`, + imageFile: '', + status: 'done', + progress: 100, + outputUrl: target.url, + originalOutputUrl: target.url, + createdAt: now, + }, + } +} + +export function resolveOpenPanelAfterLibrarySelection(currentPanel: GenerateOpenPanel): GenerateOpenPanel { + return currentPanel === 'library' ? 'library' : currentPanel +} + +function sortAssetLibraryEntries( + entries: ProjectedAssetLibraryEntry[], + sortMode: AssetLibrarySortMode, +): ProjectedAssetLibraryEntry[] { + return [...entries].sort((left, right) => compareAssetLibraryEntries(left, right, sortMode)) +} + +function compareAssetLibraryEntryGroups( + left: AssetLibraryEntryGroup, + right: AssetLibraryEntryGroup, + sortMode: Exclude, +): number { + const entryComparison = compareAssetLibraryEntries(left.entries[0], right.entries[0], sortMode) + if (entryComparison !== 0) return entryComparison + + return (ASSET_LIBRARY_CAPABILITY_ORDER.get(left.capability) ?? Number.MAX_SAFE_INTEGER) + - (ASSET_LIBRARY_CAPABILITY_ORDER.get(right.capability) ?? Number.MAX_SAFE_INTEGER) +} + +function compareAssetLibraryEntries( + left: ProjectedAssetLibraryEntry, + right: ProjectedAssetLibraryEntry, + sortMode: AssetLibrarySortMode, +): number { + if (sortMode === 'date') { + const leftTime = resolveAssetLibrarySortTimestamp(left) + const rightTime = resolveAssetLibrarySortTimestamp(right) + if (leftTime !== null && rightTime !== null && leftTime !== rightTime) return rightTime - leftTime + if (leftTime !== null && rightTime === null) return -1 + if (leftTime === null && rightTime !== null) return 1 + } + + return compareAssetLibraryEntryNames(left, right) +} + +function compareAssetLibraryEntryNames(left: ProjectedAssetLibraryEntry, right: ProjectedAssetLibraryEntry): number { + const displayNameComparison = left.displayName.localeCompare(right.displayName, undefined, { sensitivity: 'base' }) + if (displayNameComparison !== 0) return displayNameComparison + const workspacePathComparison = left.workspacePath.localeCompare(right.workspacePath, undefined, { sensitivity: 'base' }) + if (workspacePathComparison !== 0) return workspacePathComparison + return left.id.localeCompare(right.id, undefined, { sensitivity: 'base' }) +} + +function resolveAssetLibrarySortTimestamp(entry: ProjectedAssetLibraryEntry): number | null { + return parseAssetLibrarySortTimestamp(entry.createdAt) ?? parseAssetLibrarySortTimestamp(entry.updatedAt) +} + +function parseAssetLibrarySortTimestamp(value: string | undefined): number | null { + if (typeof value !== 'string' || value.length === 0) return null + const epochMs = Date.parse(value) + return Number.isFinite(epochMs) ? epochMs : null +} + +function normalizeAssetLibrarySearchQuery(searchQuery: string): string { + return searchQuery.trim().toLocaleLowerCase() +} + +function matchesAssetLibraryEntrySearch(entry: ProjectedAssetLibraryEntry, normalizedSearchQuery: string): boolean { + return [ + entry.displayName, + entry.workspacePath, + entry.source?.workspacePath, + entry.source?.displayName, + entry.manifest?.workspacePath, + entry.capability, + entry.sourceScope, + ...entry.warnings, + ] + .filter((value): value is string => typeof value === 'string' && value.length > 0) + .some((value) => matchesAssetLibrarySearch(value, normalizedSearchQuery)) +} + +function matchesAssetLibrarySearch(value: string, normalizedSearchQuery: string): boolean { + return value.toLocaleLowerCase().includes(normalizedSearchQuery) +} + +function hasInternalAssetLibraryDirectory(workspacePath: string): boolean { + const segments = workspacePath.replace(/\\/g, '/').trim().split('/').filter(Boolean) + return segments.slice(1, -1).some((segment) => segment.startsWith('.') || ASSET_LIBRARY_INTERNAL_DIRECTORY_NAMES.has(segment.toLocaleLowerCase())) +} diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/areas/generate/components/ChatPanel.tsx new file mode 100644 index 0000000..9ba89ed --- /dev/null +++ b/src/areas/generate/components/ChatPanel.tsx @@ -0,0 +1,698 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { useAppStore } from '@shared/stores/appStore' +import { useAgentStore } from '@shared/stores/agentStore' +import { useWorkflowsStore } from '@shared/stores/workflowsStore' +import { useExtensionsStore } from '@shared/stores/extensionsStore' +import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' +import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' + +// ─── Types ──────────────────────────────────────────────────────────────────── + +import type { ThinkingMode } from '@shared/stores/agentStore' +import type { Workflow } from '@shared/types/electron.d' + +interface Message { + id: string + role: 'user' | 'assistant' + content: string + thinking?: string + imageDataUrls?: string[] + actions?: ActionDone[] +} + +interface ActionDone { + tool: string + result: string + payload?: { + type: string + url?: string + face_count?: number + workflow_id?: string + workflow_name?: string + workflow?: Omit + } | null +} + +// ─── Constants ──────────────────────────────────────────────────────────────── + +const COLLAPSE_AFTER = 4 + +// ─── Prose renderer β€” basic markdown-like ──────────────────────────────────── + +function ProseMessage({ content }: { content: string }): JSX.Element { + const blocks = content.split(/\n\n+/) + return ( +
+ {blocks.map((block, i) => { + const lines = block.split('\n') + const isList = lines.every((l) => /^[-β€’*]\s/.test(l.trim()) || l.trim() === '') + if (isList) { + return ( +
    + {lines.filter(Boolean).map((l, j) => ( +
  • + β€’ + {l.replace(/^[-β€’*]\s/, '')} +
    • + ))} +
+ ) + } + return ( +

+ {block} +

+ ) + })} +
+ ) +} + +// ─── Actions card ───────────────────────────────────────────────────────────── + +const TOOL_LABELS: Record = { + decimate_mesh: 'Decimated mesh', + smooth_mesh: 'Smoothed mesh', + list_models: 'Listed models', + unload_models: 'Unloaded models', + get_mesh_info: 'Inspected mesh', + get_generation_status:'Checked generation', + list_workflows: 'Listed workflows', + run_workflow: 'Ran workflow', + create_workflow: 'Created workflow', +} + +function ActionsCard({ actions, onUndo }: { actions: ActionDone[]; onUndo?: () => void }): JSX.Element { + const [expanded, setExpanded] = useState(false) + const meshActions = actions.filter((a) => a.payload?.type === 'mesh_update') + const canUndo = meshActions.length > 0 && !!onUndo + + return ( +
+ {/* Header */} +
+ + {actions.length} action{actions.length > 1 ? 's' : ''} performed + +
+ {canUndo && ( + + )} + +
+
+ + {/* Rows */} + {expanded && ( +
+ {actions.map((a, i) => ( +
+ {TOOL_LABELS[a.tool] ?? a.tool.replace(/_/g, ' ')} + {a.payload?.type === 'mesh_update' && a.payload.face_count && ( + {a.payload.face_count.toLocaleString()} faces + )} + {a.payload?.type === 'run_workflow' && ( + {a.payload.workflow_name} + )} + {a.payload?.type === 'create_workflow' && a.payload.workflow && ( + {a.payload.workflow.name} + )} +
+ ))} +
+ )} +
+ ) +} + +// ─── Feedback row ───────────────────────────────────────────────────────────── + +function FeedbackRow({ content }: { content: string }): JSX.Element { + const [copied, setCopied] = useState(false) + + function handleCopy() { + navigator.clipboard.writeText(content) + setCopied(true) + setTimeout(() => setCopied(false), 1500) + } + + return ( +
+ + + +
+ ) +} + +// ─── Thinking block ─────────────────────────────────────────────────────────── + +function ThinkingBlock({ content }: { content: string }): JSX.Element { + const [open, setOpen] = useState(false) + return ( +
+ + {open && ( +
+

{content}

+
+ )} +
+ ) +} + +// ─── Workflow progress card ──────────────────────────────────────────────────── + +function WorkflowProgressCard({ name }: { name: string }): JSX.Element { + const runState = useWorkflowRunStore((s) => s.runState) + const pct = runState.blockProgress + return ( +
+
+
+ + {name} +
+ {pct}% +
+
+
+
+ {runState.blockStep && ( +

{runState.blockStep}

+ )} +
+ ) +} + +// ─── Main component ────────────────────────────────────────────────────────── + +export default function ChatPanel(): JSX.Element { + const { ollamaUrl, defaultModel, defaultThinking } = useAgentStore() + + const [messages, setMessages] = useState([]) + const [input, setInput] = useState('') + const [isLoading, setIsLoading] = useState(false) + const [error, setError] = useState(null) + const [showAll, setShowAll] = useState(false) + const [model, setModel] = useState(defaultModel) + const [showModelPicker, setShowModelPicker] = useState(false) + const [ollamaModels, setOllamaModels] = useState([]) + const [pendingWorkflow, setPendingWorkflow] = useState<{ id: string; name: string } | null>(null) + const [attachments, setAttachments] = useState([]) // data URLs + const [isDragging, setIsDragging] = useState(false) + const [thinkingMode, setThinkingMode] = useState(defaultThinking) + const endRef = useRef(null) + const textareaRef = useRef(null) + const modelPickerRef = useRef(null) + const fileInputRef = useRef(null) + const messagesRef = useRef([]) + messagesRef.current = messages + + const apiUrl = useAppStore((s) => s.apiUrl) + const currentJob = useAppStore((s) => s.currentJob) + const meshStats = useAppStore((s) => s.meshStats) + const updateCurrentJob = useAppStore((s) => s.updateCurrentJob) + const pushMeshUrl = useAppStore((s) => s.pushMeshUrl) + const undoMesh = useAppStore((s) => s.undoMesh) + + const workflows = useWorkflowsStore((s) => s.workflows) + const saveWorkflow = useWorkflowsStore((s) => s.save) + const setActiveWorkflow = useWorkflowsStore((s) => s.setActive) + const { modelExtensions, processExtensions } = useExtensionsStore() + const runWorkflow = useWorkflowRunStore((s) => s.run) + const runState = useWorkflowRunStore((s) => s.runState) + const allExtensions = useMemo( + () => buildAllWorkflowExtensions(modelExtensions, processExtensions), + [modelExtensions, processExtensions], + ) + + // Close model picker on outside click + useEffect(() => { + if (!showModelPicker) return + const handler = (e: MouseEvent) => { + if (modelPickerRef.current && !modelPickerRef.current.contains(e.target as Node)) + setShowModelPicker(false) + } + document.addEventListener('mousedown', handler) + return () => document.removeEventListener('mousedown', handler) + }, [showModelPicker]) + + // Watch workflow completion β†’ send follow-up to agent + useEffect(() => { + if (!pendingWorkflow) return + if (runState.status !== 'done' && runState.status !== 'error') return + + const wf = pendingWorkflow + setPendingWorkflow(null) + + if (runState.status === 'error') { + setMessages((prev) => [...prev, { + id: `sys-${Date.now()}`, + role: 'assistant', + content: `The workflow '${wf.name}' failed: ${runState.error ?? 'Unknown error'}`, + }]) + return + } + + // Update viewer with the generated mesh + if (runState.outputUrl) { + updateCurrentJob({ outputUrl: runState.outputUrl, status: 'done', progress: 100 }) + pushMeshUrl(runState.outputUrl) + } + + // Send automatic follow-up to agent + const completionCtx = `Workflow '${wf.name}' just completed.${runState.outputUrl ? ` Output mesh: ${runState.outputUrl}` : ''} Ask the user what they'd like to do next.` + callAgent(messagesRef.current, { workflowCompletion: completionCtx }) + }, [runState.status, pendingWorkflow]) + + useEffect(() => { + endRef.current?.scrollIntoView({ behavior: 'smooth' }) + }, [messages, isLoading, pendingWorkflow]) + + function buildContext(): Record { + const ctx: Record = {} + if (currentJob?.outputUrl) ctx.currentMeshPath = currentJob.outputUrl.replace('/workspace/', '') + if (meshStats?.triangles) ctx.meshTriangles = meshStats.triangles + if (workflows.length > 0) ctx.workflows = workflows.map((w) => ({ id: w.id, name: w.name })) + if (allExtensions.length > 0) ctx.extensions = allExtensions.map((e) => ({ + id: e.id, name: e.name, input: e.input, output: e.output, + })) + return ctx + } + + async function callAgent(msgs: Message[], extraContext: Record = {}) { + setIsLoading(true) + setError(null) + try { + const context = { ...buildContext(), ...extraContext } + + // Inject workflow completion as a system hint if present + const apiMessages = msgs.map((m) => { + const entry: { role: string; content: string; images?: string[] } = { + role: m.role, + content: m.content, + } + if (m.imageDataUrls?.length) { + entry.images = m.imageDataUrls.map((url) => url.split(',')[1]) + } + return entry + }) + if (extraContext.workflowCompletion) { + apiMessages.push({ role: 'user', content: `[System] ${extraContext.workflowCompletion}` }) + delete context.workflowCompletion + } + + const res = await fetch(`${apiUrl}/agent/chat`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ messages: apiMessages, ollama_url: ollamaUrl, model, context, thinking: thinkingMode }), + }) + if (!res.ok) throw new Error(`API error ${res.status}`) + + const data: { message: string; actions: ActionDone[]; thinking?: string } = await res.json() + + setMessages((prev) => [...prev, { + id: `a-${Date.now()}`, + role: 'assistant', + content: data.message, + thinking: data.thinking ?? undefined, + actions: data.actions?.length ? data.actions : undefined, + }]) + + // Extract base64 from the most recent user message that had an image attached + const latestImageDataUrl = [...msgs].reverse() + .find((m) => m.role === 'user' && m.imageDataUrls?.length) + ?.imageDataUrls?.[0] + const overrideImageData = latestImageDataUrl ? latestImageDataUrl.split(',')[1] : undefined + + for (const action of data.actions ?? []) { + if (action.payload?.type === 'mesh_update' && action.payload.url) { + updateCurrentJob({ outputUrl: action.payload.url }) + pushMeshUrl(action.payload.url) + } + if (action.payload?.type === 'run_workflow' && action.payload.workflow_id) { + const wf = workflows.find((w) => w.id === action.payload!.workflow_id) + if (wf) { runWorkflow(wf, allExtensions, overrideImageData); setPendingWorkflow({ id: wf.id, name: wf.name }) } + } + if (action.payload?.type === 'create_workflow' && action.payload.workflow) { + const draft = action.payload.workflow as Omit + const now = new Date().toISOString() + const wf: Workflow = { ...draft, id: crypto.randomUUID(), createdAt: now, updatedAt: now } + const res = await saveWorkflow(wf) + if (res.success) setActiveWorkflow(wf.id) + } + } + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e) + setError(msg.includes('fetch') ? 'Cannot reach Modly API. Is the backend running?' : msg) + } finally { + setIsLoading(false) + } + } + + async function fetchOllamaModels() { + try { + const res = await fetch(`${apiUrl}/agent/models?ollama_url=${encodeURIComponent(ollamaUrl)}`) + const data = await res.json() + setOllamaModels(data.models ?? []) + } catch { + setOllamaModels([]) + } + } + + function handleFiles(files: File[]) { + files.forEach((file) => { + if (!file.type.startsWith('image/')) return + const reader = new FileReader() + reader.onload = (e) => { + const dataUrl = e.target?.result as string + setAttachments((prev) => [...prev, dataUrl]) + } + reader.readAsDataURL(file) + }) + } + + function handleDragOver(e: React.DragEvent) { + e.preventDefault() + setIsDragging(true) + } + + function handleDragLeave(e: React.DragEvent) { + if (!(e.currentTarget as HTMLElement).contains(e.relatedTarget as Node)) setIsDragging(false) + } + + function handleDrop(e: React.DragEvent) { + e.preventDefault() + setIsDragging(false) + handleFiles(Array.from(e.dataTransfer.files)) + } + + function adjustHeight() { + const el = textareaRef.current + if (!el) return + el.style.height = 'auto' + el.style.height = `${Math.min(el.scrollHeight, 160)}px` + } + + async function handleSend() { + const text = input.trim() + if (!text || isLoading || pendingWorkflow) return + + const userMsg: Message = { + id: `u-${Date.now()}`, + role: 'user', + content: text, + ...(attachments.length ? { imageDataUrls: [...attachments] } : {}), + } + const nextMessages = [...messages, userMsg] + setMessages(nextMessages) + setInput('') + setAttachments([]) + if (textareaRef.current) textareaRef.current.style.height = 'auto' + await callAgent(nextMessages) + } + + function handleKeyDown(e: React.KeyboardEvent) { + if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); handleSend() } + } + + // Collapsed history + const collapsed = !showAll && messages.length > COLLAPSE_AFTER + const hidden = collapsed ? messages.length - COLLAPSE_AFTER : 0 + const visible = collapsed ? messages.slice(-COLLAPSE_AFTER) : messages + + return ( +
+ {/* Drag overlay */} + {isDragging && ( +
+

Drop image here

+
+ )} + + {/* Messages */} +
+ + {/* Empty state */} + {messages.length === 0 && ( +
+
+ + + + +
+

+ Ask me to generate, optimize,
or run a workflow. +

+
+ )} + + {/* Previous messages pill */} + {collapsed && ( + + )} + + {/* Message list */} +
+ {visible.map((msg) => ( +
+ {msg.role === 'user' ? ( + /* User message */ +
+ {msg.imageDataUrls && msg.imageDataUrls.length > 0 && ( +
+ {msg.imageDataUrls.map((url, i) => ( + + ))} +
+ )} +
+ {msg.content} +
+
+ ) : ( + /* Assistant message */ +
+ {msg.thinking && } + + {msg.actions && msg.actions.length > 0 && ( + + )} + +
+ )} +
+ ))} + + {/* Workflow progress card β€” visible while agent waits for workflow */} + {pendingWorkflow && } + + {/* Loading indicator */} + {isLoading && ( +
+ {[0, 1, 2].map((i) => ( + + ))} +
+ )} + + {/* Error */} + {error && ( +
+

{error}

+
+ )} + +
+
+
+ + {/* Input bar */} +
+ { if (e.target.files) handleFiles(Array.from(e.target.files)); e.target.value = '' }} + /> +
+ {/* Attachment previews */} + {attachments.length > 0 && ( +
+ {attachments.map((url, i) => ( +
+ + +
+ ))} +
+ )} +