From 489003c2cc0b326ecaaffad7ff80d6673ef9c63a Mon Sep 17 00:00:00 2001 From: nan Date: Thu, 10 Sep 2026 14:42:17 +0800 Subject: [PATCH] Add new tests and functionality for image processing and geometry handling - Introduced color_image.py for preparing standalone color images for tests. - Added hdr_image.py to create HDR textures for testing. - Implemented test_image_depth_panorama.py to evaluate panorama topology, UVs, and depth discontinuities. - Created test_shadeless.py to validate the opacity and HDR color endpoints of the Shadeless group. - Developed moge3_export.py for exporting dynamic MoGe-3 backbone and ONNX graphs. - Added material.py to inherit material slots onto generated geometry. - Built image_depth_panorama.py to construct a quad panorama with depth separation and sampling. --- experiments/panorama_surface/prototype.py | 7 +- .../add-ben2-hdr-support/.openspec.yaml | 2 + .../changes/add-ben2-hdr-support/design.md | 72 ++++ .../changes/add-ben2-hdr-support/proposal.md | 27 ++ .../specs/hdr-background-removal/spec.md | 89 +++++ .../changes/add-ben2-hdr-support/tasks.md | 31 ++ .../add-ben2-hdr-support/validation.md | 32 ++ .../changes/add-moge3-model/.openspec.yaml | 2 + openspec/changes/add-moge3-model/design.md | 63 ++++ openspec/changes/add-moge3-model/proposal.md | 23 ++ .../add-moge3-model/specs/moge3-model/spec.md | 73 ++++ openspec/changes/add-moge3-model/tasks.md | 20 + .../changes/add-moge3-model/validation.md | 51 +++ .../add-panorama-conversion/.openspec.yaml | 2 + .../changes/add-panorama-conversion/design.md | 86 +++++ .../add-panorama-conversion/proposal.md | 25 ++ .../specs/panorama-conversion/spec.md | 33 ++ .../panorama-geometry-generation/spec.md | 33 ++ .../specs/panorama-surface/spec.md | 41 +++ .../changes/add-panorama-conversion/tasks.md | 24 ++ .../.openspec.yaml | 2 + .../reuse-conversion-source-images/design.md | 55 +++ .../proposal.md | 25 ++ .../specs/conversion-source-images/spec.md | 93 +++++ .../reuse-conversion-source-images/tasks.md | 16 + .../unify-ai-input-validation/.openspec.yaml | 2 + .../unify-ai-input-validation/design.md | 83 +++++ .../unify-ai-input-validation/proposal.md | 29 ++ .../specs/ai-input-validation/spec.md | 61 +++ .../ai-operator-failure-handling/spec.md | 33 ++ .../unify-ai-input-validation/tasks.md | 30 ++ .../.openspec.yaml | 2 + .../unify-material-color-images/design.md | 71 ++++ .../unify-material-color-images/proposal.md | 28 ++ .../specs/clipboard-image-cache/spec.md | 43 +++ .../specs/material-color-images/spec.md | 82 +++++ .../unify-material-color-images/tasks.md | 25 ++ src/anyimage/assets/O_AnyImage.blend | 4 +- src/anyimage/common/ai.py | 34 +- src/anyimage/common/color_image.py | 139 +++++++ src/anyimage/common/hdr_image.py | 177 +++++++++ src/anyimage/common/image.py | 113 +----- src/anyimage/common/image_target.py | 41 ++- src/anyimage/common/material.py | 348 +++++++++--------- src/anyimage/menu.py | 5 + src/anyimage/operators/__init__.py | 2 + src/anyimage/operators/ai_setup.py | 22 +- .../operators/clipboard_image/actions.py | 55 ++- .../operators/convert_to_panorama/__init__.py | 6 + .../operators/convert_to_panorama/object.py | 71 ++++ .../convert_to_panorama/operators.py | 96 +++++ .../operators/convert_to_plane/image_plane.py | 32 +- .../operators/convert_to_plane/object.py | 87 ++--- .../operators/convert_to_plane/operators.py | 114 +++--- src/anyimage/operators/cutout_tool/object.py | 32 -- .../operators/cutout_tool/operators.py | 129 +++---- src/anyimage/operators/remove_background.py | 51 ++- src/anyimage/operators/upscale.py | 6 +- src/anyimage/properties.py | 2 +- src/anyimage/server/app.py | 6 + src/anyimage/server/geometry/depth_texture.py | 4 +- src/anyimage/server/geometry/panorama.py | 156 ++++++++ .../server/geometry/prediction_artifacts.py | 21 +- src/anyimage/server/jobs/cutout.py | 4 +- src/anyimage/server/jobs/depth_plane.py | 9 +- src/anyimage/server/jobs/panorama.py | 82 +++++ src/anyimage/server/jobs/remove_background.py | 22 +- src/anyimage/server/media/__init__.py | 2 - src/anyimage/server/media/input.py | 89 +++-- src/anyimage/server/model_catalog.py | 24 +- src/anyimage/server/model_manager.py | 11 +- src/anyimage/server/models/__init__.py | 4 +- src/anyimage/server/models/ben2.py | 61 +-- .../server/models/{moge2.py => moge.py} | 18 +- src/anyimage/server/models/onnx_ben2.py | 16 +- src/anyimage/server/models/onnx_moge2.py | 55 ++- src/anyimage/server/models/onnx_moge3.py | 159 ++++++++ src/anyimage/server/models/upscale.py | 3 +- .../common/test_alpha_presentation.py | 24 +- tests/anyimage/common/test_color_image.py | 233 ++++++++++++ tests/anyimage/common/test_hdr_image.py | 189 ++++++++++ tests/anyimage/common/test_hdr_rgb_render.py | 80 ++++ tests/anyimage/common/test_image_data.py | 21 -- tests/anyimage/common/test_image_materials.py | 9 +- tests/anyimage/common/test_image_target.py | 5 +- .../anyimage/common/test_input_preparation.py | 54 +++ tests/anyimage/common/test_material_alpha.py | 12 +- .../common/test_material_view_adaptation.py | 27 +- .../operators/test_ai_image_operators.py | 5 +- .../operators/test_ai_input_failures.py | 96 +++++ .../operators/test_clipboard_cache.py | 113 ++++++ .../operators/test_clipboard_image.py | 22 +- .../test_conversion_source_images.py | 185 ++++++++++ .../anyimage/operators/test_cutout_objects.py | 18 +- .../operators/test_cutout_operators.py | 15 +- .../anyimage/operators/test_hdr_background.py | 115 ++++++ .../operators/test_native_ai_failures.py | 77 ++++ .../operators/test_panorama_conversion.py | 188 ++++++++++ .../operators/test_plane_operators.py | 68 ++-- .../anyimage/server/test_background_alpha.py | 36 ++ tests/anyimage/server/test_edge_depth.py | 4 +- .../server/test_geometry_artifacts.py | 10 +- .../anyimage/server/test_input_validation.py | 93 +++++ tests/anyimage/server/test_job_server.py | 1 + tests/anyimage/server/test_media_input.py | 18 +- tests/anyimage/server/test_model_ben2_onnx.py | 19 +- tests/anyimage/server/test_model_geometry.py | 2 +- tests/anyimage/server/test_model_inference.py | 2 +- tests/anyimage/server/test_model_manager.py | 6 +- .../anyimage/server/test_model_moge3_onnx.py | 139 +++++++ tests/anyimage/server/test_panorama.py | 133 +++++++ tests/anyimage/server/test_server_jobs.py | 243 +++--------- tests/anyimage/test_addon_registration.py | 10 + tests/anyimage/test_image_menu.py | 1 + tests/anyimage/test_preferences.py | 6 +- tests/support/color_image.py | 11 + tests/support/hdr_image.py | 15 + tests/support/image_objects.py | 5 +- tests/tools/nodes/test_depth_relief_planes.py | 14 +- .../tools/nodes/test_image_depth_panorama.py | 259 +++++++++++++ tests/tools/nodes/test_inventory.py | 2 +- tests/tools/nodes/test_saved_layout.py | 2 +- tests/tools/nodes/test_shadeless.py | 52 +++ tests/tools/test_pack.py | 8 +- tools/models/cli.py | 9 +- tools/models/moge3_export.py | 167 +++++++++ tools/nodes/arrangement/dimensions.json | 2 + tools/nodes/build.py | 2 + tools/nodes/check.py | 3 +- tools/nodes/common/depth_surface.py | 18 +- tools/nodes/common/material.py | 22 ++ tools/nodes/groups/image_depth_cutout.py | 4 +- tools/nodes/groups/image_depth_panorama.py | 195 ++++++++++ tools/nodes/groups/image_depth_plane.py | 11 +- tools/nodes/groups/image_plane.py | 20 +- tools/nodes/groups/shadeless.py | 13 +- 136 files changed, 5758 insertions(+), 1083 deletions(-) create mode 100644 openspec/changes/add-ben2-hdr-support/.openspec.yaml create mode 100644 openspec/changes/add-ben2-hdr-support/design.md create mode 100644 openspec/changes/add-ben2-hdr-support/proposal.md create mode 100644 openspec/changes/add-ben2-hdr-support/specs/hdr-background-removal/spec.md create mode 100644 openspec/changes/add-ben2-hdr-support/tasks.md create mode 100644 openspec/changes/add-ben2-hdr-support/validation.md create mode 100644 openspec/changes/add-moge3-model/.openspec.yaml create mode 100644 openspec/changes/add-moge3-model/design.md create mode 100644 openspec/changes/add-moge3-model/proposal.md create mode 100644 openspec/changes/add-moge3-model/specs/moge3-model/spec.md create mode 100644 openspec/changes/add-moge3-model/tasks.md create mode 100644 openspec/changes/add-moge3-model/validation.md create mode 100644 openspec/changes/add-panorama-conversion/.openspec.yaml create mode 100644 openspec/changes/add-panorama-conversion/design.md create mode 100644 openspec/changes/add-panorama-conversion/proposal.md create mode 100644 openspec/changes/add-panorama-conversion/specs/panorama-conversion/spec.md create mode 100644 openspec/changes/add-panorama-conversion/specs/panorama-geometry-generation/spec.md create mode 100644 openspec/changes/add-panorama-conversion/specs/panorama-surface/spec.md create mode 100644 openspec/changes/add-panorama-conversion/tasks.md create mode 100644 openspec/changes/reuse-conversion-source-images/.openspec.yaml create mode 100644 openspec/changes/reuse-conversion-source-images/design.md create mode 100644 openspec/changes/reuse-conversion-source-images/proposal.md create mode 100644 openspec/changes/reuse-conversion-source-images/specs/conversion-source-images/spec.md create mode 100644 openspec/changes/reuse-conversion-source-images/tasks.md create mode 100644 openspec/changes/unify-ai-input-validation/.openspec.yaml create mode 100644 openspec/changes/unify-ai-input-validation/design.md create mode 100644 openspec/changes/unify-ai-input-validation/proposal.md create mode 100644 openspec/changes/unify-ai-input-validation/specs/ai-input-validation/spec.md create mode 100644 openspec/changes/unify-ai-input-validation/specs/ai-operator-failure-handling/spec.md create mode 100644 openspec/changes/unify-ai-input-validation/tasks.md create mode 100644 openspec/changes/unify-material-color-images/.openspec.yaml create mode 100644 openspec/changes/unify-material-color-images/design.md create mode 100644 openspec/changes/unify-material-color-images/proposal.md create mode 100644 openspec/changes/unify-material-color-images/specs/clipboard-image-cache/spec.md create mode 100644 openspec/changes/unify-material-color-images/specs/material-color-images/spec.md create mode 100644 openspec/changes/unify-material-color-images/tasks.md create mode 100644 src/anyimage/common/color_image.py create mode 100644 src/anyimage/common/hdr_image.py create mode 100644 src/anyimage/operators/convert_to_panorama/__init__.py create mode 100644 src/anyimage/operators/convert_to_panorama/object.py create mode 100644 src/anyimage/operators/convert_to_panorama/operators.py create mode 100644 src/anyimage/server/geometry/panorama.py create mode 100644 src/anyimage/server/jobs/panorama.py rename src/anyimage/server/models/{moge2.py => moge.py} (81%) create mode 100644 src/anyimage/server/models/onnx_moge3.py create mode 100644 tests/anyimage/common/test_color_image.py create mode 100644 tests/anyimage/common/test_hdr_image.py create mode 100644 tests/anyimage/common/test_hdr_rgb_render.py create mode 100644 tests/anyimage/common/test_input_preparation.py create mode 100644 tests/anyimage/operators/test_ai_input_failures.py create mode 100644 tests/anyimage/operators/test_clipboard_cache.py create mode 100644 tests/anyimage/operators/test_conversion_source_images.py create mode 100644 tests/anyimage/operators/test_hdr_background.py create mode 100644 tests/anyimage/operators/test_native_ai_failures.py create mode 100644 tests/anyimage/operators/test_panorama_conversion.py create mode 100644 tests/anyimage/server/test_background_alpha.py create mode 100644 tests/anyimage/server/test_input_validation.py create mode 100644 tests/anyimage/server/test_model_moge3_onnx.py create mode 100644 tests/anyimage/server/test_panorama.py create mode 100644 tests/support/color_image.py create mode 100644 tests/support/hdr_image.py create mode 100644 tests/tools/nodes/test_image_depth_panorama.py create mode 100644 tests/tools/nodes/test_shadeless.py create mode 100644 tools/models/moge3_export.py create mode 100644 tools/nodes/common/material.py create mode 100644 tools/nodes/groups/image_depth_panorama.py diff --git a/experiments/panorama_surface/prototype.py b/experiments/panorama_surface/prototype.py index 9d2209e..e2bc1cb 100644 --- a/experiments/panorama_surface/prototype.py +++ b/experiments/panorama_surface/prototype.py @@ -51,10 +51,13 @@ def remove_background(image): """Use the project's existing BEN2 implementation for the trial input.""" root = Path(__file__).resolve().parents[2] sys.path.insert(0, str(root / "src/anyimage/server/models")) - from onnx_ben2 import create_session, infer + from onnx_ben2 import create_session, infer_alpha session = create_session(root / "models/BEN2-ONNX", "cpu") - return infer(session, image, release_memory=True) + alpha = infer_alpha(session, image, release_memory=True) + foreground = image.convert("RGBA") + foreground.putalpha(Image.fromarray((alpha * 255).astype(np.uint8))) + return foreground def main(): diff --git a/openspec/changes/add-ben2-hdr-support/.openspec.yaml b/openspec/changes/add-ben2-hdr-support/.openspec.yaml new file mode 100644 index 0000000..e8cda9e --- /dev/null +++ b/openspec/changes/add-ben2-hdr-support/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-10 diff --git a/openspec/changes/add-ben2-hdr-support/design.md b/openspec/changes/add-ben2-hdr-support/design.md new file mode 100644 index 0000000..8c47a86 --- /dev/null +++ b/openspec/changes/add-ben2-hdr-support/design.md @@ -0,0 +1,72 @@ +## Context + +Remove Background 通过 ImageEditTarget 准备输入并提交结果。普通文件按路径校验，packed 图片导出 PNG；后端 BEN2 接收 Pillow RGB，预测 mask 后转换成 8-bit RGBA，Job 写入 PNG。源像素、结果载入和图片替换都涉及 HDR 精度边界。 + +BEN2 网络输出是 alpha，识别本身可复用既有模型。HDR 支持的关键是分离识别预览与原始 RGB，并让浮点信息贯穿主线程合成、EXR 编码和结果替换。 + +## Goals / Non-Goals + +本次交付为 HDR 静态图去背景：保留源图的场景线性色彩、有效动态范围和尺寸，以模型预测更新透明度。支持 Image Empty 和材质 Image Texture 两种已有入口，以及 Blender 可读取的外部、packed、generated 浮点静态图片。普通媒体沿用既有处理方式；HDR 动画在提交前明确提示首版支持范围。Upscale 的 HDR 适配作为独立能力评估。 + +## Decisions + +### 1. 在 Blender 侧捕获源图和准备预览 + +以 `Image.is_float` 及真实图片来源判断 HDR 路径，扩展名仅用于来源识别与提示。所有可读浮点静态图片走该路径，包括数值暂时处于 0–1 内的图片。准备时读取当前像素，包含尚未保存的编辑；为本次任务保存独立的 RGBA 数值快照、尺寸、色彩空间和 alpha 解释信息。 + +Blender 数据仅在主线程准备与提交阶段访问，Job 参数只包含普通字典中的路径、字符串和数值。后端始终读取 PNG 识别预览，因此公共媒体格式校验仍校验实际提交文件。源图与临时文件的所有权由准备阶段和 Job cleanup 明确交接。 + +### 2. 用固定映射生成可识别的普通 RGB + +识别预览从场景线性、非预乘 RGB 生成。Blender 的浮点缓冲已经采用场景线性颜色与预乘 alpha；预览先解关联，再采用按亮度的 Reinhard 映射，以非负亮度 Y 计算 `RGB / (1 + Y)`，执行线性 Rec.709 到 sRGB 编码并限制到可编码范围。负值只在预览中处理；源浮点数值保持原样。全黑图保持有限值。预览按源 alpha 合成到黑底，符合现有 BEN2 输入语义。 + +映射固定且不读取用户当前 View Transform、Look、Exposure，避免切换显示设置导致 mask 变化。源颜色解码由 Blender 完成。数据图片与 CHANNEL_PACKED 输入在准备时给出明确错误。 + +该映射是首版确定性基线。用实际高动态范围图片验证高光边缘、暗部与半透明物体；若识别不足，在本变更内调整并固定预览算法和对应行为测试。 + +### 3. 让 BEN2 推理核心返回数值 alpha + +```mermaid +flowchart TD + A[主线程捕获 HDR RGBA 快照] --> B[固定色调映射生成 PNG 预览] + B --> C[BEN2 推理] + C --> D[浮点 alpha 归一化并还原源尺寸] + D --> E[Job 写入 alpha.npy] + A --> F[主线程合成原始 RGB 与新 alpha] + E --> F + F --> G[写入 32-bit float RGBA EXR] + G --> H[验证并提交到原图片目标] +``` + +将现有推理核心重构为返回原尺寸 float32 alpha 数组。保留既有 mask 归一化语义，缩放过程使用浮点数据并在最后限制到 0–1，检查尺寸与有限值。普通 RGBA 消费者在自身产物边界完成量化与合成，清理旧核心入口并更新真实调用者。 + +Remove Background Job 增加明确的产物模式，例如 `output_kind: "alpha"`，HDR 静态图产出 `alpha.npy` 并通过结果字段声明。读取使用 `allow_pickle=False`，验证形状、范围和有限值。现有普通图片与动画消费者继续取得对应的 RGBA 结果。 + +### 4. 主线程合成和 EXR 持久化 + +在非预乘语义下计算 `A_out = A_source × A_predicted`，保留原始 RGB。Blender 原生浮点缓冲使用预乘 RGB，因此实现直接将 RGBA 乘以预测 alpha，保持解关联后的颜色；解码 STRAIGHT EXR 后同样采用该缓冲语义。零 alpha 像素的不可恢复颜色置零，NONE 模式按不透明源处理。测试分别比较 straight RGB 或重新合成后的可见颜色。 + +创建 `float_buffer=True` 的结果图片，使用独立临时场景指定 32-bit float RGBA EXR、ZIP 压缩，通过 `save_render` 写出场景线性值。结果采用 Linear Rec.709 与 PREMUL，保存后载入并核对数值，再打包和提交；临时场景及文件在对应阶段结束后释放。 + +复用 ImageEditTarget 和 common 内的提交、共享引用隔离、图片内容备份与恢复。HDR 的 Empty 提交同时保护材质节点等其他实际使用者。独占源图包含未保存像素或为生成图时，将其当前浮点内容转为内嵌 EXR 并建立撤销基线，保留 Image 身份；一次撤销恢复原始内容，重做恢复处理结果。提交失败时恢复操作前的源状态。先验证候选结果再提交，其他共享使用者保持原状。 + +### 5. 保持异步编辑事务完整 + +使用提交时的源快照进行合成，完成前复核原目标及图片绑定。纹理节点保存独立的 `anyimage_identity`，提交时与捕获值核对，避免 Blender 删除重建节点后复用 RNA 地址造成误认。增加对捕获后源像素或色彩解释被修改的冲突检查，发现冲突则提示重新执行并保留当前编辑。 + +临时预览由输入 cleanup 释放，alpha 由 Job 目录管理，候选 EXR 完成载入和打包后释放临时副本。取消、推理失败、合成失败、提交失败均释放快照与本次临时数据。成功结果支持一次撤销、重做以及保存重开。 + +## Risks / Trade-offs + +- 固定预览映射可能压低极暗部的可辨识度：用代表性 HDR 图片进行真实 BEN2 验收。 +- Blender 像素、alpha 模式与 EXR 编码之间存在转换：以已知数值的真实 Blender 保存重开测试约束实现。 +- 源快照增加内存占用：仅保留合成和冲突检测所需的数据，所有终止路径释放；大型输入准备失败时提供可读错误。 +- 工作区相关公共入口正在修改：实施前核对现有变更，复用最新公共校验和事务接口。 + +## Migration Plan + +先完成数值 alpha 核心与现有消费者回归，再实现 HDR 准备、合成和事务提交。验证普通媒体行为及 HDR round-trip 后完成能力交付。Job 协议由同版本插件与后端一起更新。 + +## Open Questions + +无需要用户决策的前置问题。Blender 像素空间、PREMUL round-trip 与固定预览的识别质量已通过数值测试及实际 BEN2 推理确认。 diff --git a/openspec/changes/add-ben2-hdr-support/proposal.md b/openspec/changes/add-ben2-hdr-support/proposal.md new file mode 100644 index 0000000..4414fc5 --- /dev/null +++ b/openspec/changes/add-ben2-hdr-support/proposal.md @@ -0,0 +1,27 @@ +## Why + +BEN2 通过普通 RGB 图像预测前景透明度。当前去背景流程用 Pillow 合成并保存 PNG，无法保留 HDR 图像的浮点颜色。将识别预览与原始颜色分离，可以让用户对 HDR 图片去背景，同时保留用于渲染的亮度和色彩。 + +## What Changes + +- 为 Remove Background 增加 HDR 静态图处理，覆盖 Blender 已加载的 EXR、Radiance HDR、已打包及生成的浮点图片。 +- 从提交时的浮点像素生成经过固定色调映射的 RGB 识别预览，原始颜色留在 Blender 侧。 +- BEN2 提供浮点 alpha 结果；主线程将其与源 alpha 相乘，并与原始颜色合成。 +- 结果使用 32-bit float RGBA EXR 持久化，支持保存重开、共享图片隔离、撤销和失败恢复。 +- 对 HDR 动画输入在准备阶段给出明确提示；普通图片与现有视频、序列沿用当前行为。 + +## Capabilities + +### New Capabilities +- `hdr-background-removal`: HDR 静态图片的识别预览、alpha 预测、浮点结果及生命周期。 + +### Modified Capabilities + +无已发布能力规格需要修改；与现有 ImageEditTarget 的目标提交语义衔接。 + +## Impact + +- Blender：`common/image.py`、`common/image_target.py`、`operators/remove_background.py`；复杂的 HDR 图像处理可独立放入 `common/hdr_image.py`。 +- 后端：`server/models/onnx_ben2.py`、`server/models/ben2.py`、`server/jobs/remove_background.py`，以及对应测试。 +- 输入仍为普通预览 PNG，新增 Job 的 alpha 产物模式，复用 NumPy 保存数值结果。BEN2 权重与运行依赖保持现有配置。 +- 与进行中的 `unify-ai-input-validation` 及图片目标编辑变更共同使用实施时的真实调用链。 diff --git a/openspec/changes/add-ben2-hdr-support/specs/hdr-background-removal/spec.md b/openspec/changes/add-ben2-hdr-support/specs/hdr-background-removal/spec.md new file mode 100644 index 0000000..7da5eb4 --- /dev/null +++ b/openspec/changes/add-ben2-hdr-support/specs/hdr-background-removal/spec.md @@ -0,0 +1,89 @@ +## ADDED Requirements + +### Requirement: HDR still image preparation + +系统 SHALL 为可读的浮点静态图片提供 Remove Background，覆盖 Image Empty 与材质 Image Texture 的外部、packed 和 generated 图片。输入 SHALL 采用任务开始时的当前像素快照。 + +#### Scenario: An HDR still is submitted +- **WHEN** 用户对可读的 EXR、HDR 或其他浮点静态图片执行 Remove Background +- **THEN** 系统捕获当前 RGBA、尺寸与颜色解释，并提交普通 RGB 识别预览 + +#### Scenario: Unsaved pixels are present +- **WHEN** 源浮点图包含未保存的像素编辑 +- **THEN** 识别预览与最终 RGB 均使用当前编辑后的快照 + +#### Scenario: An HDR animation is submitted +- **WHEN** 用户对 HDR 序列或浮点动画执行 Remove Background +- **THEN** 系统在创建推理任务前提示本次 HDR 功能支持静态图，并保持源图状态 + +### Requirement: Deterministic recognition preview + +系统 SHALL 通过确定性的曝光范围压缩及色彩编码生成 BEN2 的普通 RGB 预览，原始 RGB SHALL 独立保留。预览 SHALL 正确解释源 alpha，且不受用户显示变换设置影响。 + +#### Scenario: Highlights exceed one +- **WHEN** 源图包含多个大于 1 的不同亮度值 +- **THEN** 预览先进行色调映射再编码，原始快照保留这些数值 + +#### Scenario: Display settings change +- **WHEN** 相同源像素分别在不同 View Transform、Look 或 Exposure 下提交 +- **THEN** 识别预览一致 + +#### Scenario: The image is black or contains negative values +- **WHEN** 可读源像素为全黑或包含有限负颜色值 +- **THEN** 预览数值有限，源 RGB 不因预览映射改变 + +### Requirement: Float alpha result + +HDR 去背景 Job SHALL 返回原始尺寸的 float32 alpha 数组，数值有限且位于 0–1。系统 SHALL 在使用前验证产物。 + +#### Scenario: BEN2 predicts a soft edge +- **WHEN** 模型预测连续透明度边缘 +- **THEN** alpha 经过浮点缩放返回，合成前不量化为 8-bit + +#### Scenario: Alpha is invalid +- **WHEN** alpha 文件损坏、尺寸不符或包含非有限及越界数值 +- **THEN** 系统报告结果错误，保持源图片与目标绑定 + +### Requirement: HDR color and transparency persistence + +系统 SHALL 保留源 RGB 的场景线性色彩及动态范围，并以源 alpha 与预测 alpha 的乘积作为结果透明度。结果 SHALL 为同尺寸浮点图像并采用 32-bit float RGBA EXR 持久化，颜色误差 SHALL 限于浮点运算与必要颜色转换的容差。 + +#### Scenario: Straight alpha is present +- **WHEN** 源图为 straight alpha 且存在大于 1 的 RGB 和部分透明像素 +- **THEN** 结果 RGB 保留，alpha 等于源 alpha 乘预测 alpha + +#### Scenario: Premultiplied alpha is present +- **WHEN** 源图采用 PREMUL 且包含部分透明及完全透明像素 +- **THEN** 系统按 alpha 关联语义合成，边缘颜色正确且结果没有 NaN 或 Inf + +#### Scenario: The project is saved and reopened +- **WHEN** 用户保存含处理结果的 blend 文件后重新打开 +- **THEN** 结果尺寸、HDR 数值、颜色解释和透明度保持一致 + +### Requirement: Transactional target update + +系统 SHALL 复用既有图片目标提交规则，保护共享使用者并支持撤销、重做与错误恢复。Blender 数据访问 SHALL 发生于主线程，任务通信 SHALL 使用可序列化普通参数。 + +#### Scenario: Another owner shares the image +- **WHEN** 源图片存在其他实际使用者 +- **THEN** 仅发起操作的目标取得独立 HDR 结果，其他使用者仍引用原图 + +#### Scenario: The source changes during inference +- **WHEN** 原目标失效、改绑图片或源像素与颜色解释发生修改 +- **THEN** 系统拒绝覆盖当前状态，提示重新执行并清理本次临时资源 + +#### Scenario: The operation is cancelled or fails +- **WHEN** 准备、推理、合成或提交取消或失败 +- **THEN** 系统保持或恢复源图及目标绑定，释放本次快照和临时数据 + +#### Scenario: The user undoes and redoes +- **WHEN** 用户撤销并重做一次成功的 HDR 去背景 +- **THEN** 原始和处理后的浮点内容及绑定正确恢复，重做无需重新推理 + +### Requirement: Existing media behavior + +系统 SHALL 保留普通图片、视频和普通序列的现有去背景行为，所有 BEN2 消费者 SHALL 适配统一的数值 alpha 核心。 + +#### Scenario: An existing regular image or animation is processed +- **WHEN** 用户提交现有支持的普通图片或动画 +- **THEN** 系统按既有帧选择、透明度合成和输出规则完成操作 diff --git a/openspec/changes/add-ben2-hdr-support/tasks.md b/openspec/changes/add-ben2-hdr-support/tasks.md new file mode 100644 index 0000000..ed73c09 --- /dev/null +++ b/openspec/changes/add-ben2-hdr-support/tasks.md @@ -0,0 +1,31 @@ +## 1. 验证浮点图片基础行为 + +- [x] 1.1 核对最新输入校验、BEN2 消费者与图片提交调用链，保留工作区已有修改。 +- [x] 1.2 在真实 Blender 中验证已知 HDR 数值、straight/PREMUL alpha、色彩空间及 32-bit EXR 保存载入行为，确定可用颜色转换与数值容差。 +- [x] 1.3 检查图片内容备份、原位替换、打包及撤销对 float buffer 的支持，定位需要修改的公共边界。 + +## 2. 分离 BEN2 alpha 核心 + +- [x] 2.1 将推理核心改为返回原尺寸 float32 alpha，保留归一化行为，使用浮点缩放并校验有限值。 +- [x] 2.2 更新所有真实 BEN2 消费者及相关测试，在各自输出边界合成 RGBA，移除旧核心入口。 +- [x] 2.3 为 Remove Background Job 增加静态 alpha 产物模式、结果字段及安全的 NumPy 读写校验，覆盖错误与取消路径。 + +## 3. 准备 HDR 识别输入 + +- [x] 3.1 在主线程识别浮点静态图并捕获当前 RGBA 快照，覆盖外部、packed、generated 和未保存像素。 +- [x] 3.2 实现固定色调映射及 sRGB 预览编码，覆盖 alpha、负值、全黑及显示设置独立性。 +- [x] 3.3 接入 Remove Background 的准备与普通参数提交，明确拒绝 HDR 动画并验证临时输入清理。 + +## 4. 合成与提交 HDR 结果 + +- [x] 4.1 校验 float alpha 并按 straight/PREMUL 语义合成，保留源 HDR RGB 和尺寸。 +- [x] 4.2 创建浮点结果并写入无损 32-bit RGBA EXR，完成载入、打包及临时文件释放。 +- [x] 4.3 复用并补全 ImageEditTarget 的浮点提交与恢复行为，覆盖共享隔离、独占 Image 身份、源修改冲突、撤销和重做。 +- [x] 4.4 验证取消、失败、目标失效各路径的资源释放和源状态恢复。 + +## 5. 验收 + +- [x] 5.1 更新并运行相关后端、common 和 Operator 测试，覆盖浮点 alpha 精度与现有普通媒体行为。 +- [x] 5.2 用已知 RGB 大于 1、有限负值、软 alpha 的图片进行真实 Blender 两种入口及保存重开验证；记录数值误差。 +- [x] 5.3 用代表性 HDR 实图运行 BEN2，检查高光、暗部、半透明边缘与黑底预览效果，必要时调整固定映射并补齐验证。 +- [x] 5.4 运行全量 `uv run pytest` 并确认正常退出，检查最终差异及旧入口引用，记录验证结果。 diff --git a/openspec/changes/add-ben2-hdr-support/validation.md b/openspec/changes/add-ben2-hdr-support/validation.md new file mode 100644 index 0000000..a2efd0a --- /dev/null +++ b/openspec/changes/add-ben2-hdr-support/validation.md @@ -0,0 +1,32 @@ +## 验证环境 + +2026-09-10，Windows，项目 uv 环境，Blender Python 4.5.3 LTS，BEN2 本地 `models/BEN2-ONNX/onnx/model_fp16.onnx`，ONNX Runtime CPU。 + +## 浮点与编辑事务 + +- 真实 Blender 验证外部 EXR、packed、generated、未保存像素，覆盖 Image Empty 与材质 Image Texture，以及共享和独占图片。 +- RGBA 样本包含大于 1 的颜色、有限负值、零 alpha 与部分透明像素；32-bit ZIP EXR 保存、打包、载入和 blend 保存重开均通过数值检查。 +- 单独验证 STRAIGHT EXR 解码后的预乘缓冲语义，处理后解关联颜色在 `rtol=1e-6` 内保持一致。 +- 已验证 Radiance `.hdr` 实际文件读取与识别预览生成，源浮点像素保持不变。 +- 原生 Operator 的撤销、重做、取消、提交异常和资源释放通过测试；未保存的独占浮点源图采用内嵌 EXR 撤销基线。 +- 纹理节点删除重建测试覆盖独立身份字段，防止 RNA 地址复用造成异步目标误认。 + +## 实际 BEN2 推理 + +Cycles 渲染 320×320 金属 Suzanne，16 samples，两盏面光源，原始 RGB 峰值约 112.432。用渲染 alpha 作为轮廓参考，构造三档曝光的灰底 HDR 图，以及保留原始软 alpha 的透明图。四组均执行真实 BEN2 Job、浮点 alpha 读回、Blender 合成及 EXR 持久化。 + +| 输入 | 原始 RGB 峰值 | 输出预乘 RGB 峰值 | 轮廓 IoU | +| --- | ---: | ---: | ---: | +| 正常亮度 | 112.4321 | 112.4235 | 0.9822 | +| 16 倍亮度 | 1798.9137 | 1796.3881 | 0.9874 | +| 0.05 倍亮度 | 5.6216 | 5.6176 | 0.9757 | +| 原始透明度 | 112.4321 | 112.4292 | 0.9656 | + +IoU 使用预测 alpha 与渲染 alpha 的 0.5 阈值轮廓计算。四组 EXR 读回值相对于 `source_rgba × predicted_alpha` 的最大绝对误差均为 0。已目视检查高亮预览、低亮度预览与预测轮廓。该结果验证当前渲染样例的识别效果；不同实图的抠图质量取决于模型预测。 + +## 自动检查 + +- OpenSpec 严格校验通过。 +- BEN2 的旧 `infer` 入口及旧节点使用者检查名称已清理，相关调用者和测试已更新。 +- 最终 `uv run pytest -q`：1074 passed、2 xfailed、52 subtests passed，175.71 秒，退出码 0。两项 xfail 为已有的浮点图片显示色彩空间适配测试。 +- `git diff --check` 通过。 diff --git a/openspec/changes/add-moge3-model/.openspec.yaml b/openspec/changes/add-moge3-model/.openspec.yaml new file mode 100644 index 0000000..e8cda9e --- /dev/null +++ b/openspec/changes/add-moge3-model/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-10 diff --git a/openspec/changes/add-moge3-model/design.md b/openspec/changes/add-moge3-model/design.md new file mode 100644 index 0000000..ee3a110 --- /dev/null +++ b/openspec/changes/add-moge3-model/design.md @@ -0,0 +1,63 @@ +## Context + +项目现有几何推理使用 ONNX Runtime，Windows 使用 DirectML，其他平台使用默认 Runtime。MoGe 3 的稀疏细化已在项目外改写为标准 ONNX 索引、聚合和矩阵运算，完整三次细化已实际运行。 + +实现使用上游提交 `74fbce054ebed49800de42d0ad0e83495065719a`，骨干与细化分别导出为 ONNX 图。骨干支持动态输入和 token 数，细化支持动态点数。在 RTX 5070 Ti 上，1200–3600 tokens 的完整动态推理约需 4.86–17.11 秒，平均深度相对误差约 0.002%，二值 Mask 完全一致。参考比较关闭 FlexGEMM 独立的 TF32 开关。Windows DirectML 与 Linux CPU 均已通过真实 Job 验证，详细结果见 [验收记录](validation.md)。 + +实验脚本与日志位于项目外 `C:/Users/icrdr/AppData/Local/Temp/anyimage-moge3-trial/`。正式转换位于 `tools/models/moge3_export.py`，模型准备和运行使用项目已有的资产机制。 + +## Goals + +以最小改动将完整 MoGe 3 接入 MoGe 2 的现有工程流程，沿用输入准备、参数、下载、会话管理和几何产物契约。 + +## Decisions + +### 动态尺寸与分辨率等级作为接入门槛 + +正式模型接收现有输入准备流程产生的图片尺寸及宽高比，沿用现有 resolution_level 到 token 数的控制方式，输出尺寸对应分析图片。256×192 / 1200 tokens 是原型测试配置，正式推理需通过动态形状及可变 token 数验证，同一组模型资产在任务间接受不同配置。 + +沿用已验证的两个图边界：骨干输出初始坐标、编码特征、法线、Mask 和尺度；细化图输出 log-depth 增量。MoGe 3 模块内部使用 NumPy 生成体素邻域并执行三轮调度，学习网络计算全部由 ONNX Runtime 执行，几何恢复复用现有后处理能力。逐层对比用于验证动态化后的数值一致性。 + +正式 Runtime 使用标准 ONNX 图和现有执行提供程序，完整执行三次细化。原型中主要网络计算由 DirectML 执行，部分索引与计数节点由 ONNX Runtime CPU 执行；正式验收继续记录实际执行路径。 + +### 最小工程改动 + +沿用现有图像准备、最大分析尺寸、分辨率等级、设备选择、Job 参数和产物。新模型特有的图集合、索引生成及迭代封装在 `models/onnx_moge3.py`；已有推理调用点只增加必要的模型分派。 + +复用现有模型下载与会话生命周期，通用编排名称仅在职责确实扩展时局部调整并一次更新真实调用者。Operator、Blender 响应和节点资产继续消费现有契约。正式运行依赖保持现有 ONNX 和后处理体系，离线转换依赖独立管理。 + +### 精度和性能验证 + +比较同权重、相同输入与分辨率等级的原始三次细化结果和 ONNX 结果。先用 FP32 验证语义，再评估 FP16。实验需在导出前声明验收容差，记录有效区域深度与点图误差、法线角误差、内参偏差、Mask 一致性及每次细化误差，不得根据结果事后放宽。 + +样本包含室内、细小物体、室外、不同宽高比及全景透视视图；覆盖低/高分辨率等级、自动 FOV 与固定 90 度 FOV。记录转换耗时、ONNX 文件大小、会话启动、热推理耗时及峰值内存/显存。Windows 验证 CPU 和 DirectML 的完整执行情况及实际算子回退；其他平台需分别验证，不沿用 PyTorch 的 CUDA 平台限制。 + +### 验证后的正式接入 + +```mermaid +flowchart TD + A[已验证的双 ONNX 图原型] --> B[动态尺寸及分辨率等级验证] + B --> C{精度 性能 提供程序通过} + C -->|通过| D[生成并校验模型资产] + D --> E[登记模型与下载] + E --> F[统一 MoGe ONNX 推理入口] + F --> G[深度平面 裁切 全景] + C -->|未通过| H[记录阻碍与实验结果] +``` + +转换工具放在 `tools/models`，依赖固定版本或上游提交；最终登记所有图文件及 external data 的大小和哈希，完整下载后标记就绪。镜像发布后才登记对应来源。生产推理只读取已准备的 ONNX 资产。 + +新增模型沿用现有几何模型枚举，保留已有模型键和默认值；仅调整识别新模型所必需的元数据与筛选。输出使用现有 CPU NumPy 数组和相机内参约定。全景传入 90 度 FOV，保持 Mask 与源有效性、Alpha 合成以及径向距离融合契约。 + +缓存按模型、设备和图集合识别，连续帧及视图复用会话，切换模型和关闭时释放资源。取消与失败沿用现有 Job 事务。 + +## Risks / Trade-offs + +- 动态体素分箱可能放大浮点数值差异 → 对相同 FP32 配置按预定容差比较。 +- 高分辨率等级增加计算量与内存 → 已记录 CPU/DirectML 性能及峰值占用，继续沿用现有分辨率等级控制。 +- 多图执行会增加 CPU/GPU 数据搬运 → 测量完整流水线耗时并据此决定图边界。 +- 提供程序算子覆盖不同 → Windows DirectML 与 Linux CPU 已验收，macOS 仍需实机验证。 + +## Migration Plan + +先基于已完成原型验证动态尺寸、分辨率等级、精度与性能，再完成转换工具和模型登记。沿用现有设置与默认模型，只增加新模型选择及必要分派。实施更新相关测试，检查依赖与打包内容一致性。 diff --git a/openspec/changes/add-moge3-model/proposal.md b/openspec/changes/add-moge3-model/proposal.md new file mode 100644 index 0000000..4b9fa98 --- /dev/null +++ b/openspec/changes/add-moge3-model/proposal.md @@ -0,0 +1,23 @@ +## Why + +MoGe-3 ViT-L 的完整三次细化已在项目外通过 CPU ONNX Runtime 与 DirectML 原型验证。正式接入应沿用 MoGe 2 的工程流程，以最小改动增加模型选项，并完成动态输入和分辨率等级验证。 + +## What Changes + +- 将已跑通的双 ONNX 图原型扩展到现有输入尺寸处理、宽高比和分辨率等级，建立可复现导出与精度比较流程。 +- 验证通过后新增 `MOGE3_VITL` 模型选项、ONNX 资产下载和就绪检查，保留已有模型及默认值。 +- 深度平面、裁切与全景统一按模型分派，推理统一采用现有 ONNX Runtime，复用几何产物。 +- 两个 ONNX 图、体素邻域生成与三次细化封装在 MoGe 3 模型内部；复用现有 Operator、节点资产、Job 参数和用户操作流程。 +- 以完整细化、数值一致性、执行提供程序兼容性与可用性能作为正式接入门槛。 + +## Capabilities + +### New Capabilities + +- `moge3-model`: 完整 MoGe 3 的 ONNX 推理、模型选择、资产准备与几何工作流集成。 + +### Modified Capabilities + +## Impact + +改动集中在离线转换工具、模型登记、MoGe 3 ONNX 模块和必要的推理分派；模型枚举、下载与会话管理沿用既有机制。PyTorch、MoGe、FlexGEMM 与 Triton 仅用于隔离的离线转换及参考验证环境；正式 Server 沿用 ONNX Runtime 与现有后处理依赖。正式接入以动态尺寸、分辨率等级及完整流程验收通过为前提。 diff --git a/openspec/changes/add-moge3-model/specs/moge3-model/spec.md b/openspec/changes/add-moge3-model/specs/moge3-model/spec.md new file mode 100644 index 0000000..a6a7f45 --- /dev/null +++ b/openspec/changes/add-moge3-model/specs/moge3-model/spec.md @@ -0,0 +1,73 @@ +## Purpose + +让用户通过现有 ONNX Runtime 体系使用完整的 MoGe-3 ViT-L 几何预测，使模型准备、选择和执行在深度平面、裁切与全景中具有一致行为，并通过完整细化及数值验证保证结果可信。 + +## ADDED Requirements + +### Requirement: Execute complete MoGe 3 with ONNX + +正式推理 SHALL 通过 ONNX Runtime 执行 MoGe 3 的学习网络计算，并完成三次细化；运行环境 SHALL 无需 PyTorch、FlexGEMM 或 Triton。 + +#### Scenario: Production inference +- **WHEN** 在仅安装声明的 ONNX 运行依赖的 Server 中执行 MoGe 3 +- **THEN** 完成包含三次细化的预测，不依赖外部 Torch 环境或服务 + +### Requirement: Validate conversion before availability + +系统 SHALL 在完整流程的动态尺寸、分辨率等级、精度、性能和执行提供程序验证通过后才将新模型登记为正式可用模型。 + +#### Scenario: Incomplete conversion +- **WHEN** 只有关闭细化的模型可以运行，或完整模型未通过预定数值验收 +- **THEN** 保留验证结果且不将其作为完整 MoGe 3 发布 + +### Requirement: Select and prepare model + +系统 SHALL 提供 MOGE3_VITL 选项，保留已有模型键和默认值，下载并校验推理所需的全部 ONNX 资产。 + +#### Scenario: Download success +- **WHEN** 所需图文件及外部权重全部下载并通过完整性校验 +- **THEN** 标记模型就绪，深度平面、裁切与全景使用用户所选模型 + +#### Scenario: Interrupted download +- **WHEN** 下载失败、取消或文件不完整 +- **THEN** 不标记就绪，并允许重新准备 + +### Requirement: Preserve geometry contract + +MoGe 3 SHALL 按所选分辨率等级提供与分析图尺寸对应的深度、法线、有效性、点图和相机内参，并遵循既有源 Alpha 处理契约。 + +#### Scenario: Image inference +- **WHEN** 有效图片完成推理 +- **THEN** 有效区域深度为有限正值，产物可供既有几何工作流应用 + +#### Scenario: Panorama inference +- **WHEN** 全景任务选择 MoGe 3 +- **THEN** 透视视图使用已知 90 度水平 FOV，模型 Mask 与源有效性共同约束融合及 Alpha 输出 + +### Requirement: Reuse existing input and workflow controls + +MoGe 3 SHALL 沿用 MoGe 2 的输入准备、最大分析尺寸、分辨率等级及设备选择流程。同一组已下载模型资产 SHALL 接受现有流程产生的不同图片尺寸、宽高比和分辨率等级，输出尺寸对应分析图片。 + +#### Scenario: Vary image dimensions +- **WHEN** 用户连续提交现有输入流程支持的不同尺寸和宽高比图片 +- **THEN** 使用同一组模型资产完成预测，遵循现有分析尺寸限制及输出尺寸契约 + +#### Scenario: Change resolution level +- **WHEN** 用户修改现有 MoGe Resolution Level 后再次执行任务 +- **THEN** 新任务采用对应 token 数并执行三次细化，复用已下载模型资产 + +#### Scenario: Switch to MoGe 3 in an existing workflow +- **WHEN** 用户在深度平面、裁切或全景工作流中将模型改为 MoGe 3 +- **THEN** 沿用原有任务参数、操作步骤与几何产物，模型内部完成多图及细化调度 + +### Requirement: Manage supported sessions + +系统 SHALL 按已验证的 ONNX 设备能力执行，复用连续帧或视图的会话，在模型切换及关闭时释放资源，对不支持的执行方式和推理失败给出明确反馈。 + +#### Scenario: Model switching +- **WHEN** 用户在 MoGe 2 与 MoGe 3 之间切换 +- **THEN** 执行新选择并释放旧模型资源 + +#### Scenario: Unsupported execution or cancellation +- **WHEN** 执行设备不支持、资源不足或任务取消 +- **THEN** 给出明确状态，不应用部分结果，并允许后续重新运行 diff --git a/openspec/changes/add-moge3-model/tasks.md b/openspec/changes/add-moge3-model/tasks.md new file mode 100644 index 0000000..68587ee --- /dev/null +++ b/openspec/changes/add-moge3-model/tasks.md @@ -0,0 +1,20 @@ +## 1. 原型与动态配置验收 + +- [x] 1.1 在项目外将完整 MoGe 3 拆为骨干与细化 ONNX，使用固定 256×192 / 1200 tokens 在 CPU 和 DirectML 上运行三张图、三次细化，保存参考比较及耗时记录。 +- [x] 1.2 将骨干扩展为动态输入尺寸和可变 token 数，用同一组 ONNX 资产连续运行不同宽高比、分析尺寸与分辨率等级，验证输出尺寸遵循现有契约。 +- [x] 1.3 定位原型中较大像素误差，确定后续验收容差，在 CPU 与 DirectML 验证动态配置及 90 度 FOV，记录精度、实际执行路径、耗时与内存峰值。 +- [x] 1.4 汇总动态配置、精度、性能及平台支持矩阵；完整验收通过后进入第 2 阶段，未通过项保留具体实验结论。 + +## 2. 模型资产与推理 + +- [x] 2.1 将已验证的转换方案整理至 tools/models，固定离线依赖，重新导出并通过相同精度验收以验证可复现性。 +- [x] 2.2 登记 MOGE3_VITL 的全部 ONNX 图及外部数据文件，验证大小和哈希，测试下载成功、失败、取消与就绪判断。 +- [x] 2.3 将双图、邻域生成和三次细化封装在 MoGe 3 ONNX 模块，仅增加必要推理分派；测试现有参数、输出尺寸、内参、有效性及分辨率等级传递。 +- [x] 2.4 更新会话缓存和释放，测试模型交替、连续帧复用与失败后恢复，确认正式环境不需 Torch 或 Triton。 + +## 3. 工作流接入与验收 + +- [x] 3.1 在现有模型登记与枚举中加入新选项，测试旧模型键和默认值保持、新模型选择及设备错误提示，核对继续使用既有 Job 参数。 +- [x] 3.2 在深度平面、裁切与全景的必要调用点接入分派，复用现有 Operator 和节点资产，运行产物、90 度 FOV、Alpha、取消与融合回归测试。 +- [ ] 3.3 在正式 ONNX Runtime 环境运行完整工作流并记录结果，在支持矩阵中的其他平台完成对应验收。（Windows 与 Linux 已通过，macOS 待实机验收，见 validation.md。） +- [x] 3.4 检查依赖、Manifest 和打包测试的一致性，运行相关测试与全量 pytest，确认正常退出及最终差异范围。 diff --git a/openspec/changes/add-moge3-model/validation.md b/openspec/changes/add-moge3-model/validation.md new file mode 100644 index 0000000..c06f8a3 --- /dev/null +++ b/openspec/changes/add-moge3-model/validation.md @@ -0,0 +1,51 @@ +# MoGe 3 ONNX 验收 + +2026-09-10。新增 `MOGE3_VITL`，复用 MoGe 2 的输入准备、0–9 分辨率等级、下载、会话缓存和几何产物。骨干支持动态图片尺寸及 token 数，细化图支持动态点数，正式推理执行三次细化。 + +## 资产 + +转换源码：`tools/models/moge3_export.py`。上游提交为 `74fbce054ebed49800de42d0ad0e83495065719a`，权重 revision 为 `184008f877d7ad1ad4c2cd2182a9bd1f63d0e5be`。固定 Torch、ONNX、NumPy 和上游提交后重复导出，两次文件哈希一致。 + +| 文件 | 字节数 | SHA-256 | +|---|---:|---| +| backbone.onnx | 1324190318 | f21246c6921373c8c79ecdb8c78818a79bd2506c2fb88f4d4cdec21b92f725e3 | +| refiner.onnx | 157729628 | 67ad7297aba18f8ef7939e8f700a5b25c41a19ddb68656bc2cd08b54c248fe2a | + +权重内嵌于两个 ONNX 文件。镜像已发布到 `models.omoolab.xyz/moge-3-vitl-onnx/`，使用项目 `ModelManager.download` 下载到新目录，耗时 136.1 秒，大小、哈希及 `ready()` 校验通过。 + +## 精度与动态配置 + +参考为原始 MoGe 3、相同权重和三次细化。上游 FlexGEMM 的 `SPCONV_ALLOW_TF32` 独立于 Torch TF32 设置；关闭该选项后进行 FP32 比较。验收脚本预设平均深度相对误差小于 0.1%、99 分位小于 1%、二值 Mask 一致率大于 99.9%。 + +同一组动态图依次执行以下配置；耗时包含骨干、邻域生成、三次细化和后处理，不包含会话创建。 + +| 图片宽×高 | tokens | 水平 FOV | Windows CPU | DirectML | DirectML 平均深度相对误差 | 99 分位 | +|---|---:|---|---:|---:|---:|---:| +| 384×256 | 1200 | 自动 | 10.65 s | 4.86 s | 0.00216% | 0.02068% | +| 257×385 | 2400 | 90° | 22.92 s | 9.67 s | 0.00228% | 0.01823% | +| 320×320 | 3600 | 自动 | 36.41 s | 17.11 s | 0.00167% | 0.01609% | + +三项均通过 CPU 和 DirectML 容差，二值 Mask 一致率为 100%。正式模块使用现有 MoGe 2 后处理，等级 0/9 的平均点图相对误差约 0.00217%/0.00173%，平均法线角误差约 0.006°，归一化内参最大绝对偏差小于 0.000002。2400 tokens 是图层测试值；公开等级继续按原公式映射至 1200–3600 tokens。 + +## 工作流与平台 + +| 平台 / Runtime | 验收结果 | +|---|---| +| Windows / DirectML 1.24.4 / RTX 5070 Ti | 动态精度、深度、裁切、12 视图全景通过；纯 ONNX 环境 | +| Windows / CPU | 动态精度通过；原型 CPU Runtime 为 1.29.0 | +| Linux x64 / CPU 1.24.4 / Python 3.12 | 使用项目声明的正式依赖，深度、裁切、12 视图全景通过 | +| macOS / CPU、CoreML | 当前没有可用设备，待实机验收 | + +真实 Job 调用仅替换进度/取消传输上下文，图片准备、模型管理、推理、几何处理与文件写入均使用项目代码。Windows 深度、裁切、全景分别约 9.81、5.45、67.13 秒；Linux 分别约 19.19、9.10、112.10 秒。深度/裁切生成 EXR、元数据及法线，全景完成 12 个 90° 视图推理和径向融合并生成 EXR 与元数据。 + +DirectML 的主要网络计算在 GPU 执行，部分 Gather、Unsqueeze、Concat、Less、Cast、ReduceSum 由 ONNX Runtime CPU 提供程序执行。正式环境依赖保持原样，未引入 Torch、FlexGEMM 或 Triton。 + +等级 0/9 连续运行时，Windows 进程峰值工作集约 1483 MiB；显卡整体占用由 6370 MiB 上升至最高 15716 MiB（每 0.5 秒采样，包含桌面及其他进程，不能视为模型独占显存）。Linux 三个 Job 进程峰值 RSS 约 4664 MiB。高等级的内存占用明显高于低等级。 + +## 回归与复现记录 + +全量 pytest：1143 passed、52 subtests passed，187.79 秒，正常退出。随后新增会话加载失败恢复测试，MoGe 3 专项 7 passed。覆盖下载完整性、缓存复用/切换/恢复、三次细化、尺寸、参数及现有 Alpha、FOV、取消、全景融合和打包内容契约。OpenSpec 严格校验通过。 + +实验与运行日志保存在 `C:/Users/icrdr/AppData/Local/Temp/anyimage-moge3-trial/`：`dynamic_reference.py`、`dynamic_validate.py`、`production_validate.py`、`job_validate.py`、`linux_jobs.py`、`download_validate.py`、`memory_validate.py`，及对应 JSON/日志。Linux 使用只读项目挂载的 Python 3.12 容器；Windows 全量测试使用按项目锁文件创建的独立测试环境。 + +任务 3.3 保留未完成状态，待补充 macOS 实机验收。 diff --git a/openspec/changes/add-panorama-conversion/.openspec.yaml b/openspec/changes/add-panorama-conversion/.openspec.yaml new file mode 100644 index 0000000..1ea7e36 --- /dev/null +++ b/openspec/changes/add-panorama-conversion/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-09 diff --git a/openspec/changes/add-panorama-conversion/design.md b/openspec/changes/add-panorama-conversion/design.md new file mode 100644 index 0000000..86a9fa7 --- /dev/null +++ b/openspec/changes/add-panorama-conversion/design.md @@ -0,0 +1,86 @@ +## Context + +独立原型验证了 12 个透视视图的 MoGe2 推理、距离融合和 Quad Sphere。正式转换沿用 Image Empty 的异步转换流程。用户明确选择通过现有 Remove Background 预先处理背景，转换消费当前图片和 Alpha。 + +## Goals / Non-Goals + +本次范围是静态 2:1 等距柱状全景的菜单转换、正式节点资产和 Split Threshold。沿用原型的 Subdivide、Depth Scale 和单张深度图片约定。 + +## Decisions + +### 入口与对象生命周期 + +```mermaid +flowchart TD + A[Image Empty 右键 AnyImage] --> B[Convert to Panorama] + B --> C[检查静态图和 2 比 1 尺寸] + C --> D[读取当前 RGBA 和 Preferences] + D --> E[提交全景 Job] + E --> F[加载并打包颜色和深度] + F --> G[创建全景对象和节点修改器] + G --> H[替换源 Empty 并记录撤销] +``` + +新增 `operators/convert_to_panorama/`，`operators.py` 定义 `ConvertToPanorama` 与 `GeneratePanorama`，`object.py` 创建结果，`__init__.py` 导出注册清单。公开入口为 `anyimage.convert_to_panorama`，Job 为 `generate-panorama-geometry`。复用 `common` 的输入暂存、材质、深度加载、修改器赋值和对象收尾函数。 + +模型、推理级别和分析尺寸从 Preferences 读取，Job 参数为普通字典。Blender 数据只在主线程读取输入和处理响应。新对象中心对应原 Empty 原点，继承其世界变换；局部 +Z 为全景上方，U=0 朝向 +X，U 增加沿 +Y 转动。生成成功后替换源对象并激活结果；失败、取消或源对象已失效时清理本次临时资源。图片显示偏移只属于平面预览，球面方向由完整图片 UV 定义。 + +### 后端与数据协议 + +```mermaid +flowchart TD + A[当前 RGBA 全景] --> B[12 个重叠 90 度视图] + B --> C[MoGe2 与已知 FOV 的几何恢复] + C --> D[有效视图的径向距离] + D --> E[横向周期的对数梯度融合] + E --> F[尺度锚定与有限距离场] + A --> G[源 Alpha 对齐到深度尺寸] + F --> H[depth.exr] + G --> H +``` + +`server/jobs/panorama.py` 负责任务调度，`server/geometry/panorama.py` 负责投影与融合。将已知 FOV 接入已有 ONNX 后处理和模型会话生命周期，普通图片继续使用焦距估计。保留原分辨率颜色；视图分析尺寸遵循输入上限，首版目标视图为 512×512，融合网格最长边上限 512 且保持 2:1。Subdivide 独立控制最终网格密度。 + +源 Alpha 对齐、黑底模型输入及可见区域排除采用原型已验证的流程。全透明视图跳过推理，部分视图无效时继续融合其余有效观测；全局无有效几何时报错。逐视图及融合前后检查取消，释放模型资源。 + +| 字段 | 文件 | 内容 | +| --- | --- | --- | +| 颜色文件 | `image.0001.png` | 当前 RGBA，保留源 Alpha | +| `depth` | `depth.exr` | float32 RGB 重复保存径向距离，A 为融合 Mask × 源 Alpha | +| `depth_metadata` | `depth.json` | `projection: equirectangular`、`image_size`、`reference_distance` | + +`reference_distance` 为满足 Alpha 阈值的有限正径向距离中位数，用于 Split Threshold 的尺度归一化。全景元数据按投影类型单独读取；复用现有 EXR 写入及 Non-Color、CHANNEL_PACKED、Pack 配置。颜色由材质采样，几何节点只有一个深度图片输入。 + +### Quad Sphere 与 Split Threshold + +新增资产 `O Image Panorama`。Quad Sphere 与球面校正子组的构建定义归入本仓库节点构建模块，参考已检查的 `O Quad Sphere`，运行时从 `O_AnyImage.blend` 加载。面数为 `6 × 4^Subdivide`，默认 Subdivide=7；Depth Scale 默认 1，范围 0–1；Validity Threshold 默认 0.9；Split Threshold 默认 0，范围 0–1。深度图片、材质和参考距离作为数据输入。 + +```mermaid +flowchart TD + A[可调细分 Quad Sphere] --> B[固定单位射线与球面 UV] + B --> C[按深度 EXR Alpha 剔除无效面] + C --> D[相邻面径向距离变化判断] + D --> E[Split Edges 与条带面清理] + E --> F[分离面角沿原射线取所属面距离] + F --> G[Depth Scale 插值到重建表面] + G --> H[材质与临时属性清理] +``` + +提取并复用 `tools/nodes/common/depth_surface.py` 中的相邻面采样、内边判断、Split Edges、面角归属和条带面清理。共用逻辑消费明确的深度字段、基础面中心和归一化比例；Depth Plane 的调用同步适配，保持已有求值结果。 + +对全景，令相邻面距离为 rA/rB，单位球面上的面中心为 cA/cB，参考距离为 R。沿用现有 2.5 系数，以 `abs(rA-rB) / R × Split Threshold > 2.5 × distance(cA,cB)` 选择有两个邻面的内边。阈值为 0 时直接保留通过有效性剔除的原网格；阈值增大时选择更多断层边。 + +选中断层附近的面角使用其所属面的距离，未选中的顶点保持原采样；所有点沿原始球面射线定位。最终半径为 `1 + Depth Scale × (目标距离 - 1)`。条带面清理遵循当前 Depth Plane 的同一拓扑快照规则。 + +全景面中心采样由面中心方向重新计算 UV，横向循环采样；材质面角 UV 独立处理经度分支和极点。这样左右接缝与普通内边参与同样的判断。分裂后保留原射线及面角 UV，末尾清理专用临时属性。 + +### 验证与交付 + +正式模块从原型迁移有效算法及测试，运行时依赖正式模块与打包资产。此次实现运行相关单元测试、完整回归测试和独立 Blender 求值检查。节点资产更新走现有离线资产流程；扩展包及文档构建属于另行发布操作。 + +## Risks / Trade-offs + +- 径向距离与平面 Z 深度的比较尺度不同 → 用参考距离归一化，并测试相同形状整体缩放后的分裂选择一致。 +- 全景接缝和极点的 UV 有多种等价表示 → 用方向生成采样坐标，并验证接缝两侧的等价深度断层。 +- 共享分裂代码近期增加了条带清理 → 保留现有 Depth Plane 回归结果，并覆盖全景分裂后孤立条带、Alpha 边缘和连续斜面。 +- 天空及遮挡导致预测缺失 → 以当前源图 Alpha 和融合 Mask 确定有效几何，允许通过现有 Remove Background 预先处理。 diff --git a/openspec/changes/add-panorama-conversion/proposal.md b/openspec/changes/add-panorama-conversion/proposal.md new file mode 100644 index 0000000..c3d7fe8 --- /dev/null +++ b/openspec/changes/add-panorama-conversion/proposal.md @@ -0,0 +1,25 @@ +## Why + +全景原型已验证 Quad Sphere、径向深度和单张深度 EXR 的可行性。现在需要把生成流程接入 Image Empty 的常用入口，并通过 Depth Plane 的 Split Threshold 机制处理前后景连接处的拉伸。 + +## What Changes + +- 在 Image Empty 的 AnyImage 右键菜单增加 `Convert to Panorama`，通过后台 Job 将静态 2:1 全景转换成可编辑的全景表面。 +- 将原型的多视图 MoGe2 推理、距离融合和深度 Alpha 导出接入正式 Server。 +- 新增 `O Image Panorama` 节点资产，提供 Quad Sphere 细分、Depth Scale、Validity Threshold 和 Split Threshold。 +- 复用 Depth Plane 的分边、面角深度重定位和条带面清理机制，按全景径向距离适配判断。 +- 延续单深度图片输入：`depth.exr` 的 Alpha 在导出时合成模型 Mask 与当前源图 Alpha。 + +## Capabilities + +### New Capabilities + +- `panorama-conversion`: Image Empty 菜单、异步转换、对象替换与撤销。 +- `panorama-geometry-generation`: 全景多视图推理、融合与文件产物。 +- `panorama-surface`: Quad Sphere、球面 UV、深度有效性剔除与 Split Threshold。 + +### Modified Capabilities + +## Impact + +涉及 `menu.py`、类型注册、新增 `operators/convert_to_panorama/`、Server Job 与全景几何模块、共享深度表面节点构建函数，以及对应测试。生产节点组由资产构建流程写入 `O_AnyImage.blend`，运行时按名称加载。使用现有 MoGe2、ONNX Runtime、NumPy 与 SciPy 环境。 diff --git a/openspec/changes/add-panorama-conversion/specs/panorama-conversion/spec.md b/openspec/changes/add-panorama-conversion/specs/panorama-conversion/spec.md new file mode 100644 index 0000000..dd3bf17 --- /dev/null +++ b/openspec/changes/add-panorama-conversion/specs/panorama-conversion/spec.md @@ -0,0 +1,33 @@ +## ADDED Requirements + +### Requirement: Image Empty panorama conversion entry + +系统 SHALL 在 Image Empty 的 AnyImage 右键菜单展示 `Convert to Panorama`，调用 `anyimage.convert_to_panorama`。 + +#### Scenario: Select panorama conversion +- **WHEN** 用户选中静态 2:1 Image Empty 并选择 Convert to Panorama +- **THEN** 提交当前图片及 Alpha 的全景生成任务，使用 Preferences 中的模型、推理级别和分析尺寸 + +#### Scenario: Unsupported image +- **WHEN** 输入是动画、视频或非 2:1 图片 +- **THEN** 提示输入限制并保留源对象 + +### Requirement: Consume current image alpha + +转换 SHALL 直接使用当前源图及 Alpha。背景处理由用户通过现有 Remove Background 完成。 + +#### Scenario: Already removed background +- **WHEN** 源 Empty 的天空已透明 +- **THEN** 当前 Alpha 进入全景任务，转换过程中只运行几何模型 + +### Requirement: Complete conversion transaction + +系统 SHALL 在主线程加载结果、打包贴图并创建节点对象，成功后替换源 Empty，并提供完整撤销。 + +#### Scenario: Successful conversion +- **WHEN** Job 返回有效产物 +- **THEN** 创建使用 O Image Panorama 的对象，继承源 Empty 世界变换并以其原点为拍摄中心，激活结果并支持撤销恢复源对象 + +#### Scenario: Cancelled or failed conversion +- **WHEN** 任务失败、取消或响应时源对象已失效 +- **THEN** 清理本任务暂存及未使用数据块，不应用部分转换结果 diff --git a/openspec/changes/add-panorama-conversion/specs/panorama-geometry-generation/spec.md b/openspec/changes/add-panorama-conversion/specs/panorama-geometry-generation/spec.md new file mode 100644 index 0000000..513859b --- /dev/null +++ b/openspec/changes/add-panorama-conversion/specs/panorama-geometry-generation/spec.md @@ -0,0 +1,33 @@ +## ADDED Requirements + +### Requirement: Generate panoramic radial distance + +Server SHALL 从当前 RGBA 全景生成 12 个重叠的 90 度透视视图，使用已知 FOV 恢复 MoGe2 几何并融合为横向周期的径向距离场。 + +#### Scenario: Continuous panorama +- **WHEN** 输入包含跨越 U=0/1 的连续表面 +- **THEN** 两侧使用同一周期求解，产出有限正距离且无额外拼接断层 + +#### Scenario: Invalid views and cancellation +- **WHEN** 部分视图全透明或模型无有效预测 +- **THEN** 利用其余有效观测完成融合，全局无有效几何时返回明确错误，并在视图与融合阶段响应取消 + +### Requirement: Export combined depth alpha + +系统 SHALL 返回颜色文件、`depth.exr` 与 `depth.json`。深度 RGB 保存径向距离，Alpha 在导出时一次合成融合 Mask × 源 Alpha；元数据 SHALL 标明等距柱状投影、深度图尺寸和参考距离。 + +#### Scenario: Transparent background +- **WHEN** 源图某区域 Alpha 为零 +- **THEN** 该区域深度 EXR Alpha 为零，颜色文件保留源 Alpha + +#### Scenario: Fractional alpha +- **WHEN** 有效模型 Mask 为 1 且源 Alpha 为 0.5 +- **THEN** 深度导出 Alpha 为 0.5，几何端直接消费该值 + +### Requirement: Use managed production runtime + +全景生成 SHALL 使用现有模型会话、Preferences 配置和 Job 生命周期；Job 参数可序列化，Server 与 Blender 数据保持进程边界。 + +#### Scenario: Configured inference +- **WHEN** 用户修改模型或推理分辨率级别后发起转换 +- **THEN** 新任务采用配置值，最终网格细分仍由独立 Subdivide 参数控制 diff --git a/openspec/changes/add-panorama-conversion/specs/panorama-surface/spec.md b/openspec/changes/add-panorama-conversion/specs/panorama-surface/spec.md new file mode 100644 index 0000000..ce5ad4f --- /dev/null +++ b/openspec/changes/add-panorama-conversion/specs/panorama-surface/spec.md @@ -0,0 +1,41 @@ +## ADDED Requirements + +### Requirement: Editable Quad Sphere surface + +系统 SHALL 提供资产 `O Image Panorama`，使用全四边面 Quad Sphere，并提供 Subdivide、Depth Scale、Validity Threshold 和 Split Threshold。几何节点 SHALL 只有一个深度图片输入。 + +#### Scenario: Sphere and depth endpoints +- **WHEN** 所有样本有效且 Split Threshold 为 0 +- **THEN** Subdivide=n 产生 6×4^n 个四边面，Depth Scale=0 为单位球，Depth Scale=1 为预测径向距离表面 + +#### Scenario: Validity culling +- **WHEN** 深度 EXR Alpha 低于 Validity Threshold +- **THEN** 按 Depth Plane 的点域布尔字段与面域删除方式剔除对应区域 + +### Requirement: Depth Plane split semantics + +全景 SHALL 复用 Depth Plane 的相邻面深度差判断、内边分裂、面角深度重定位和条带面清理机制，并使用参考距离归一化的径向距离适配球面。 + +#### Scenario: Disabled splitting +- **WHEN** Split Threshold 为 0 +- **THEN** 分裂与条带清理完全旁路，有效性剔除保持生效 + +#### Scenario: Foreground depth discontinuity +- **WHEN** 相邻有效面的归一化距离跳变达到分裂条件 +- **THEN** 分离选中的内边，断层附近面角沿原球面射线使用所属面的距离，未选中顶点保持原始深度采样 + +#### Scenario: Threshold and scale consistency +- **WHEN** 增大 Split Threshold 或对所有距离与参考距离同时乘以相同正比例 +- **THEN** 增大阈值不会减少被选中的断层边，等比例距离变化不会改变断层边选择 + +#### Scenario: Seam and strip cleanup +- **WHEN** 深度断层跨越全景经度接缝或分裂产生孤立条带 +- **THEN** 经度接缝采用周期采样，面角 UV 保持正确，条带面按 Depth Plane 的同一拓扑快照规则清理 + +### Requirement: Packaged node assets + +生产节点组及 Quad Sphere 子组 SHALL 包含在 `O_AnyImage.blend` 中，通过现有加载入口使用。 + +#### Scenario: Reopen converted object +- **WHEN** 在新 Blender 进程重新打开转换后的文件 +- **THEN** 无需访问原型目录或外部参考库即可求值，贴图、细分、Depth Scale 和 Split Threshold 均正常工作 diff --git a/openspec/changes/add-panorama-conversion/tasks.md b/openspec/changes/add-panorama-conversion/tasks.md new file mode 100644 index 0000000..e0eb3cf --- /dev/null +++ b/openspec/changes/add-panorama-conversion/tasks.md @@ -0,0 +1,24 @@ +## 1. 全景后端 + +- [x] 1.1 将原型的投影、周期距离融合和参考距离计算迁移到正式 Server 模块，保留解析几何测试 +- [x] 1.2 为现有 MoGe2 ONNX 推理增加已知 FOV 的几何恢复，复用模型会话并验证普通图片回归 +- [x] 1.3 实现 generate-panorama-geometry Job、尺寸和有效性检查、取消与清理，导出当前颜色、深度及全景元数据 +- [x] 1.4 验证源 Alpha 与 Mask 仅在深度导出合成一次，覆盖透明视图、全局无有效数据和跨经度接缝 + +## 2. 全景节点资产 + +- [x] 2.1 在本仓库实现 Quad Sphere 及校正子组的资产构建，创建 O Image Panorama 的细分、球面 UV 和 Depth Scale +- [x] 2.2 复用深度 EXR 加载与 Alpha 剔除，保持几何节点单深度图片输入 +- [x] 2.3 提取共享 Split Threshold 机制并同步适配现有调用，将径向距离判断、面角重定位和条带清理接入全景 +- [x] 2.4 覆盖阈值旁路、断层强度、尺度不变性、连续斜面、经度接缝、极点及条带清理，并运行现有深度表面回归 + +## 3. 菜单与对象转换 + +- [x] 3.1 实现 ConvertToPanorama 与 GeneratePanorama，接入 Preferences、输入暂存、Job 参数与注册清单 +- [x] 3.2 在 Image Empty 的 AnyImage 菜单添加 Convert to Panorama，验证静态 2:1 输入限制及当前 Alpha 传递 +- [x] 3.3 实现主线程结果加载、材质、全景对象和修改器，覆盖源变换继承、替换、撤销及失败清理 + +## 4. 集成验证 + +- [x] 4.1 通过既有离线节点资产流程更新 O_AnyImage.blend，验证新进程加载和打包后独立求值 +- [x] 4.2 使用已去天空的真实全景与解析样本检查正式菜单流程和 Split Threshold，运行完整测试并检查最终差异 diff --git a/openspec/changes/reuse-conversion-source-images/.openspec.yaml b/openspec/changes/reuse-conversion-source-images/.openspec.yaml new file mode 100644 index 0000000..e8cda9e --- /dev/null +++ b/openspec/changes/reuse-conversion-source-images/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-10 diff --git a/openspec/changes/reuse-conversion-source-images/design.md b/openspec/changes/reuse-conversion-source-images/design.md new file mode 100644 index 0000000..ee9de9a --- /dev/null +++ b/openspec/changes/reuse-conversion-source-images/design.md @@ -0,0 +1,55 @@ +## Context + +动机见 proposal.md。Plane 与 Panorama 复制源 Image，Depth / Relief 从 Job 颜色文件重新加载。`configure_material_color_image()` 还会按 Image users 再次复制。图片编辑工具已通过 `has_other_image_empty()` 判断同一 Image 数据块的其他 Empty 引用。 + +## Goals / Non-Goals + +设计目标是把图像所有权选择集中在转换准备阶段，使材质配置、对象替换与错误恢复使用同一个确定的 Image。转换范围为 Plane、Depth Plane、Relief Plane 和 Panorama。 + +## Decisions + +### 1. 复用统一的 Empty 引用判断 + +在主线程接收转换结果时，使用 `has_other_image_empty()` 检查整个 `bpy.data.objects`，排除正在转换的源对象。判断依据为 Image 数据块身份。 + +```mermaid +flowchart TD + A[验证源 Image Empty] --> B{其他 Empty 引用同一 Image} + B -->|有| C[复制原图并保留未保存像素] + B -->|无| D[复用原 Image] + C --> E[应用材质颜色空间与 Alpha 规则] + D --> E + E --> F[创建材质和转换对象] + F --> G[完成对象替换与撤销记录] +``` + +原 Image 名称、路径和内容来源保留；副本沿用原名并交给 Blender 消歧。复制数据块仍指向同一源文件，按既有资源生命周期保证 generated、dirty 和 packed 图像的内容可保存读取。 + +### 2. 所有权选择与材质配置分离 + +在 common 中提供四种转换共用的图像准备入口。材质配置函数原地配置已选定的 Image，并返回同一个实例。逐一检查 `create_image_material()` 的调用方，让需要独立结果的入口在配置前准备好 Image。 + +普通转换继续采用现有适配开关、inverse 色彩空间映射、sRGB 回退及对应 STRAIGHT / PREMUL 模式。Panorama 的 shadeless float / 非 sRGB 保留分支继续使用源图配置。切换设置时保留业务像素，包括透明区域 RGB 和未保存像素。 + +其他材质用户随复用 Image 接受配置变化；这落实与图片编辑工具一致的共享规则。相关转换测试以本变更规格为准，协调 `configure-material-view-adaptation` 中较宽的用户隔离预期。 + +### 3. 原图承担颜色来源 + +Depth / Relief 回调直接准备源 Image，Job 保留深度、法线和 metadata 产物。删除该调用链独占的 `load_color_result_image()`、`copy_color_images()` 及导出和测试引用。`color_data_name()` 是否保留由 Cutout 等实际调用决定。 + +Panorama Job 移除冗余 `image.0001.png` 保存和清理条目。原图与深度允许各自使用既有分辨率，颜色通过现有 UV 采样。Plane 继续从源 Empty 传递动画帧设置。 + +### 4. 转换包含失败恢复 + +修改复用 Image 前保存必要的颜色空间、Alpha、像素和相关状态；对象替换成功前发生异常时恢复。清理由本次操作创建的对象、网格、材质和 Image 副本负责，原 Image 始终保留。沿用并验证 Blender 的 Undo / Redo，使成功转换能够恢复 Empty 及原图设置。异步结果应用时重新验证源对象及其图像身份，源图已替换的结果应取消应用，避免颜色与深度来源不一致。 + +## Risks / Trade-offs + +- [已有材质共享原图配置变化] → 作为明确的共享语义提供行为测试。 +- [切换颜色空间导致 dirty / generated 像素重读] → 复用 common 像素保存逻辑，并验证透明 RGB、HDR 与保存重载。 +- [材质创建失败后原图已被修改] → 将配置和对象创建纳入同一恢复范围，覆盖配置成功后发生异常的路径。 +- [共享配置函数影响其他入口] → 审查所有材质调用点，验证 Cutout 与剪贴板材质行为。 + +## Migration Plan + +按公共图像准备、转换接入、后端产物清理的顺序实施，再运行相关测试和全量测试。已有文件在用户再次转换时采用新规则。全部修改集中在源码与测试，可通过回退本次实现恢复原行为。 diff --git a/openspec/changes/reuse-conversion-source-images/proposal.md b/openspec/changes/reuse-conversion-source-images/proposal.md new file mode 100644 index 0000000..47bd074 --- /dev/null +++ b/openspec/changes/reuse-conversion-source-images/proposal.md @@ -0,0 +1,25 @@ +## Why + +Convert To 当前复制或重新加载颜色贴图，产生额外的 `_color` Image 和后端颜色文件。统一采用图片编辑工具的共享 Empty 判断，可以直接复用原图，并保留材质所需的颜色空间与 Alpha 配置。 + +## What Changes + +- Plane、Depth Plane、Relief Plane、Panorama 在没有其他 Image Empty 引用原图时直接复用原 Image；存在其他 Empty 引用时复制 Image 数据块。 +- 原图保留名称和路径；副本沿用原名并使用 Blender 自动消歧命名。材质颜色直接取自原图。 +- 由转换入口决定 Image 所有权，再应用现有颜色空间、Alpha 和 Panorama HDR 保留规则。 +- 共享判断仅考虑其他 Image Empty；已有材质引用同一原图时同步看到配置变化。 +- 清理 Depth / Relief 和 Panorama 的冗余后端颜色输出与对应加载路径，补充失败恢复和像素保留验证。 + +## Capabilities + +### New Capabilities + +- `conversion-source-images`: 四种 Convert To 的原图复用、共享隔离、材质配置与结果生命周期。 + +### Modified Capabilities + +## Impact + +涉及 `common/image.py`、`common/material.py`、两个转换 Operator 包、Server 的 depth_plane / panorama Job、颜色文件辅助函数及相关测试。复用现有依赖和节点资产。 + +当前 `openspec/specs` 尚无已归档规格。本变更明确细化 `configure-material-view-adaptation` 中转换入口的共享策略：其他材质用户可以共享配置变化，其他 Image Empty 必须隔离；实施时应协调相关测试预期。 diff --git a/openspec/changes/reuse-conversion-source-images/specs/conversion-source-images/spec.md b/openspec/changes/reuse-conversion-source-images/specs/conversion-source-images/spec.md new file mode 100644 index 0000000..175d95a --- /dev/null +++ b/openspec/changes/reuse-conversion-source-images/specs/conversion-source-images/spec.md @@ -0,0 +1,93 @@ +## Purpose + +规定 Plane、Depth Plane、Relief Plane 和 Panorama 转换时颜色贴图的来源、共享引用与配置生命周期，使用户能够复用原始 Image，并在其他 Image Empty 仍然使用原图时获得独立且配置正确的材质图像。 + +## ADDED Requirements + +### Requirement: Conversion selects images by other Empty references + +四种 Convert To SHALL 在应用转换时检查整个 Blender 数据中的其他 Image Empty 是否引用同一源 Image。没有其他引用时 SHALL 复用原 Image；存在时 SHALL 使用原 Image 的独立副本。 + +#### Scenario: Only the converted Empty uses the source + +- **WHEN** 源 Image 没有被其他 Image Empty 引用 +- **THEN** 转换材质 SHALL 引用原 Image 数据块并保留其名称和路径 + +#### Scenario: Another Empty shares the image + +- **WHEN** 另一个 Image Empty 引用同一 Image，包括隐藏对象或其他场景对象 +- **THEN** 材质 SHALL 使用独立副本，沿用原名并由 Blender 消歧 +- **AND** 其他 Empty 的 Image 引用、像素、颜色空间和 Alpha 模式 SHALL 保持原值 + +#### Scenario: Existing materials share the image + +- **WHEN** 原 Image 被材质引用且没有其他 Image Empty 引用 +- **THEN** 转换 SHALL 复用原 Image，已有材质 SHALL 同步看到该 Image 的配置变化 + +#### Scenario: Separate images have the same file path + +- **WHEN** 另一 Empty 使用路径相同但数据块不同的 Image +- **THEN** 该引用 SHALL 视为独立图像，转换 SHALL 复用当前源 Image + +### Requirement: Conversion preserves material presentation rules + +转换 SHALL 对选定的 Image 应用既有材质颜色空间和 Alpha 配对规则，保留业务 RGB、Alpha、透明区域 RGB、精度和未保存像素。Panorama 的 float 或非 sRGB shadeless 图像 SHALL 沿用源颜色空间和 Alpha 模式。 + +#### Scenario: Standard material adaptation is applied + +- **WHEN** 普通转换关闭适配，或所需 inverse 空间回退到 sRGB +- **THEN** 选定 Image SHALL 使用 sRGB + PREMUL + +#### Scenario: Inverse adaptation is available + +- **WHEN** 普通转换启用适配且对应 inverse 空间可用 +- **THEN** 选定 Image SHALL 使用该空间与 STRAIGHT + +#### Scenario: Panorama preserves HDR interpretation + +- **WHEN** Panorama 源图为 float 或采用现有保留分支匹配的非 sRGB 空间 +- **THEN** 材质 Image SHALL 保留原颜色空间、Alpha 模式与 HDR 像素 + +#### Scenario: Unsaved and packed pixels remain readable + +- **WHEN** generated、dirty 或 packed 图像完成转换并保存重载 +- **THEN** 图像业务像素 SHALL 在原有编码精度内保留，包括完全透明区域的 RGB + +### Requirement: Conversion uses source color content directly + +转换材质 SHALL 使用源图内容与分辨率。Depth / Relief 和 Panorama Job SHALL 仅输出各自所需的几何数据产物；转换 SHALL 无需产生额外颜色文件。 + +#### Scenario: Depth or relief result is applied + +- **WHEN** Job 返回深度、法线和 metadata +- **THEN** 转换 SHALL 使用原图或其独立副本完成材质绑定 + +#### Scenario: Panorama result is applied + +- **WHEN** Job 返回深度和 metadata +- **THEN** 转换 SHALL 使用源全景颜色完成材质绑定 + +#### Scenario: Plane converts an animated image + +- **WHEN** Plane 转换 movie 或 sequence 图像 +- **THEN** 材质 SHALL 保留源 Empty 的帧起点、偏移、时长和循环设置 + +### Requirement: Conversion preserves source state on failure and undo + +转换 SHALL 在失败时恢复复用原图的有效配置和像素，释放本次创建的临时资源，并保留源 Empty。成功转换 SHALL 支持 Undo / Redo。 + +#### Scenario: Material or object creation fails + +- **WHEN** 原图已配置但转换在完成对象替换前失败 +- **THEN** 原图的颜色空间、Alpha、业务像素和源 Empty 引用 SHALL 恢复 +- **AND** 本次创建的未使用资源 SHALL 清理 + +#### Scenario: The source changes during an asynchronous job + +- **WHEN** 结果应用时源对象失效或其 Image 已替换 +- **THEN** 转换 SHALL 取消应用并保留当前源状态 + +#### Scenario: A completed conversion is undone and redone + +- **WHEN** 用户撤销成功的转换再重做 +- **THEN** 撤销 SHALL 恢复源 Empty 和转换前图像状态，重做 SHALL 恢复转换对象及对应材质配置 diff --git a/openspec/changes/reuse-conversion-source-images/tasks.md b/openspec/changes/reuse-conversion-source-images/tasks.md new file mode 100644 index 0000000..803c070 --- /dev/null +++ b/openspec/changes/reuse-conversion-source-images/tasks.md @@ -0,0 +1,16 @@ +## 1. 公共图像准备与材质配置 + +- [x] 1.1 在 common 中统一转换图像准备，复用其他 Empty 引用判断；测试覆盖独占、共享 Empty、隐藏或跨场景 Empty、仅材质共享和同路径不同 Image，并检查原名与副本命名。 +- [x] 1.2 将材质配置改为配置已选定的 Image，审查全部材质创建调用点；运行材质测试，验证配置后 Image 身份、inverse / sRGB 回退、Alpha 配对及 Cutout、剪贴板行为。 +- [x] 1.3 复用公共像素和状态辅助函数实现转换失败恢复；测试配置失败及配置成功后材质或对象创建失败，确认原图状态恢复与临时资源清理。 + +## 2. 转换入口和后端产物 + +- [x] 2.1 接入 Plane、Depth Plane、Relief Plane 的原图准备逻辑，保留 Plane 动画设置，并验证异步源对象与 Image 身份；通过转换测试确认引用身份、原始分辨率、帧参数和源图替换时取消应用。 +- [x] 2.2 接入 Panorama，保留 HDR / 非 sRGB 分支和可保存的源像素；运行全景转换测试，覆盖独占与共享 Image、HDR 数值及 Alpha 模式。 +- [x] 2.3 移除 Depth / Relief 和 Panorama 的冗余颜色输出及专用加载、复制辅助函数，清理导出和对应测试；通过 Job 测试验证产物集合，并用 rg 确认删除符号无残留业务引用。 + +## 3. 集成验证 + +- [x] 3.1 验证 byte、float、generated、dirty、packed 图像的透明 RGB 和保存重载行为，覆盖复用与复制两条路径，并验证转换 Undo / Redo 恢复对象及图像配置。Panorama HDR 往返通过；普通材质 float 使用 display 色彩空间重载的两项既有 xfail 保留。 +- [x] 3.2 运行相关测试与 `uv run pytest`，确认正常退出；检查最终差异仅包含本变更实施内容，已有工作区修改保留，旧颜色路径与测试预期完成清理。全量结果：1125 passed、2 xfailed、52 subtests passed，退出码 0。 diff --git a/openspec/changes/unify-ai-input-validation/.openspec.yaml b/openspec/changes/unify-ai-input-validation/.openspec.yaml new file mode 100644 index 0000000..1ea7e36 --- /dev/null +++ b/openspec/changes/unify-ai-input-validation/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-09 diff --git a/openspec/changes/unify-ai-input-validation/design.md b/openspec/changes/unify-ai-input-validation/design.md new file mode 100644 index 0000000..c3c5516 --- /dev/null +++ b/openspec/changes/unify-ai-input-validation/design.md @@ -0,0 +1,83 @@ +## Context + +以当前工作区调用链为依据： + +| 位置 | 发现 | 处理方向 | +| --- | --- | --- | +| `common/ai.py` | BEN2 专属扩展名与后端重复，Upscale 只检查路径存在 | 统一公共预检 | +| `server/media/input.py` | 扩展名提示混排；目录过滤后仅提示没有帧 | 按错误原因和媒体类别提示 | +| `server/models/ben2.py` | 单帧校验使用第三套提示 | 复用图片格式校验 | +| `common/image.py` | 路径文案限定 Image Empty；序列直接复制；packed 导出失败可能留下目录 | 使用 source 文案、复制前校验、准备失败释放资源 | +| Remove BG、Depth/Relief Plane | 内部 Operator 调用位于异常处理之外 | 捕获同步调用错误并取消 | +| `operators/ai_setup.py`、`invoke_ai_setup_if_needed()` | 下载、安装与设置入口嵌套调用 | 在公共准备入口和设置 Operator 内处理失败 | +| Panorama | 已捕获嵌套调用错误并清理 | 回归现有行为 | + +BlendJob 的 `request()` 错误会被转换为 ERROR report 和 CANCELLED；Blender 的 `bpy.ops` 调用仍可能将该 report 抛为 RuntimeError。异步 Job 失败已有集中处理。前后端可以共享 `server` 内纯 Python 模块，已有 resolution 模块采用这一依赖方向；后端保持可独立加载。 + +## Goals / Non-Goals + +统一 AI 输入校验、用户可读的错误原因、失败取消状态和临时输入生命周期。支持格式保持 JPG/JPEG/PNG/WEBP 与现有九种视频扩展名；各 Operator 继续表达自己的静态图、尺寸与比例限制。自动转码、扩展格式支持和 BlendJob 框架重构作为独立需求处理。 + +## Decisions + +### 1. 公共媒体模块维护格式规则与提示 + +在 `server/media/input.py` 提供轻量路径/格式校验，按图片、视频或帧目录识别输入，图片模型使用相同规则的图片子集。模块加载和预检仅依赖标准库。 + +`common/ai.py` 的 `require_input_path()` 负责 Blender 路径解析并调用该规则；删除 BEN2 专属校验及扩展名常量。`prepare_input()` 在后端再次执行校验，应对独立 Job 请求和提交后源文件变化。前端和后端使用同一消息生成位置，比只在后端检查更早反馈，也避免复制列表。 + +```mermaid +flowchart TD + Source[Blender 源图片] --> Prepare[解析路径与准备输入] + Prepare --> Validate[公共媒体校验] + Validate --> Submit[提交 Job] + Submit --> Recheck[后端公共媒体校验] + Recheck --> Decode[读取图片或视频帧] + Decode --> Model[模型处理] + Validate --> Error[普通错误提示与取消] + Recheck --> Error + Decode --> Error +``` + +### 2. 提示由原因决定，格式按类别列出 + +保留当前英文 UI 语言，示例模板： + +- `Source image file does not exist: {path}` / `Source video file does not exist: {path}`。 +- `Unsupported input format: {suffix}. Supported images: {images}. Supported videos: {videos}.` 扩展名排序且小写；无扩展名使用 `(no extension)`。 +- `No supported image frames found in: {path}. Supported images: {images}.`。 +- `Unable to read image: {path}` / `Unable to read video: {path}`，底层原因保留异常链供诊断。 + +只将路径检查、文件读取与解码边界的预期异常转换为输入提示，模型推理或资源错误保留实际原因。通用输入阶段保持图片与视频类别一致；BEN2 单帧模型内部使用图片子集提示。 + +### 3. 明确目录与 Blender 序列的差别 + +Blender SEQUENCE 在创建临时目录前验证源扩展名属于支持的图片格式，不支持时直接报格式错误。已识别的同名序列按原排序和帧区间处理。 + +通用目录仍仅选择支持格式的图片文件并忽略其他文件；预检与后端使用同一收集规则。全无有效图片时报告目录及支持图片格式。这一提案修复 Blender 序列被误报为空目录的问题，混合目录按筛选契约验收。 + +格式校验针对实际提交文件。packed 图片和像素裁剪输出使用现有 PNG 导出路径，普通文件继续使用原始文件；测试应明确两条路径的预期，避免把源名称当作实际编码。`IMAGE_NAME_SUFFIXES` 是命名识别列表，继续按命名职责维护。 + +### 4. 在拥有调用与资源的边界处理失败 + +Remove BG、Depth/Relief Plane、AI Environment 的嵌套 `bpy.ops` 调用捕获预期 RuntimeError；涉及输入准备的边界同时处理 OSError/ValueError。通过已有 report 出口返回 CANCELLED，规范 Blender 添加的首个 `Error: ` 前缀与尾部换行，保留实际原因。公共 AI setup 入口复用同一报错约定；异步 Job 仍由 BlendJob 管理。 + +临时输入在准备完成前由准备函数拥有；Job 成功接管后由 Job cleanup 拥有。准备失败、同步调用抛错、返回 CANCELLED 和异步失败分别验证释放责任。packed 导出失败应清理整个临时目录。清理只能触及本次操作创建的输入，不影响用户原文件和源图片引用。 + +优先在现有公共函数与直接调用处落实职责；复用消息格式化即可，保持各业务 Operator 的调用和资源归属可见。 + +## Risks / Trade-offs + +- 前后端文件状态可能变化 → 后端复核并提供同一原因分类。 +- 测试 mock 不会自动模拟 Blender report 抛错 → 单元测试注入 RuntimeError，补充真实 Blender 入口验证。 +- 调用层次多可能重复 report → 以最终用户提示无重复前缀、无 Python traceback 为验收点，检查 Blender UI 实际输出。 +- 解码错误与推理错误容易混淆 → 只在读取边界包装解码异常。 +- 其他未归档变更修改相关入口 → 以实施时工作区为基线，保留 Undo、源图共享引用及 Panorama 行为。 + +## Migration Plan + +依次落地公共校验、消费者接入、调用边界与清理，再运行相关及全量测试。删除旧 BEN2 校验入口并检查引用。变更不涉及持久数据迁移；必要时可整体回退本变更代码与测试。 + +## Open Questions + +无实施前置问题。真实 Blender 下 report 的重复展示情况列为实施验收项。 diff --git a/openspec/changes/unify-ai-input-validation/proposal.md b/openspec/changes/unify-ai-input-validation/proposal.md new file mode 100644 index 0000000..713207e --- /dev/null +++ b/openspec/changes/unify-ai-input-validation/proposal.md @@ -0,0 +1,29 @@ +## Why + +Remove BG 与 Upscale 对同一种输入给出不同的格式提示，Remove BG 的普通校验错误还会经嵌套 Operator 调用变成 Python traceback。检查发现格式规则重复、序列过滤、路径提示和临时输入清理也存在同一调用链上的不一致，需要建立统一的输入与失败处理契约。 + +## What Changes + +- AI 文件输入使用同一套格式规则与错误文案，前端提前校验，后端独立复核；错误说明实际扩展名及支持的图片、视频格式。 +- 区分路径缺失、格式不支持、目录无有效帧和文件无法解码，路径提示统一使用 source image/video，适用于 Image Empty 和材质 Image Texture。 +- Blender 图片序列在复制前验证格式，避免不支持的序列被当成空目录；通用帧目录明确过滤规则。 +- Remove BG、Depth/Relief Plane 与 AI Environment 的嵌套调用按普通错误取消操作，并统一资源释放；审查 Panorama 既有处理作为回归范围。 +- 保留静态图、尺寸、比例等功能专属限制，统一其异常出口；格式能力以当前实际提交文件为准。 + +## Capabilities + +### New Capabilities + +- `ai-input-validation`: AI 文件、帧目录与 Blender 图片序列的公共校验、错误分类及文案。 +- `ai-operator-failure-handling`: AI Operator 的失败展示、取消状态与临时输入所有权。 + +### Modified Capabilities + +无。当前 `openspec/specs/` 为空。本提案与现有 `add-texture-node-image-actions`、`add-panorama-conversion` 变更共同验收相关入口。 + +## Impact + +- `src/anyimage/server/media/input.py`、`server/media/__init__.py`、`server/models/ben2.py`：公共格式契约、目录处理与模型输入校验。 +- `src/anyimage/common/ai.py`、`common/image.py`：前置校验、源路径与序列准备、临时资源释放。 +- `operators/remove_background.py`、`upscale.py`、`convert_to_plane/operators.py`、`ai_setup.py` 及其他公共入口消费者：错误传播与回归。 +- 更新相关行为测试，运行全量 pytest，并用 Blender 验证嵌套调用的实际报错。依赖与节点资产沿用当前版本。 diff --git a/openspec/changes/unify-ai-input-validation/specs/ai-input-validation/spec.md b/openspec/changes/unify-ai-input-validation/specs/ai-input-validation/spec.md new file mode 100644 index 0000000..51bdd93 --- /dev/null +++ b/openspec/changes/unify-ai-input-validation/specs/ai-input-validation/spec.md @@ -0,0 +1,61 @@ +## ADDED Requirements + +### Requirement: Shared input format validation + +AI 输入 SHALL 使用公共格式规则，在 Blender 提交前与后端处理前校验实际输入路径。支持图片为 JPG/JPEG/PNG/WEBP，视频沿用当前 MP4/MOV/AVI/MKV/WEBM/M4V/MTS/MPG/MPEG，扩展名匹配忽略大小写。 + +#### Scenario: Unsupported file in image edits +- **WHEN** Remove BG 与 Upscale 使用同一个存在的 `.tiff` 普通文件，且环境和模型已就绪 +- **THEN** 两者在提交 Job 前取消，展示相同格式提示，包含 `.tiff` 和分类排序的支持格式 + +#### Scenario: Direct backend input +- **WHEN** 后端直接收到不支持格式的文件或无扩展名文件 +- **THEN** 使用与前端相同的格式提示规则，无扩展名显示 `(no extension)` + +#### Scenario: Supported uppercase extension +- **WHEN** 实际输入为存在的 `.PNG` 或 `.MP4` 文件 +- **THEN** 通过对应图片或视频格式校验 + +#### Scenario: Model image validation +- **WHEN** BEN2 单帧入口收到不支持的图片扩展名 +- **THEN** 使用公共格式提示的图片子集，并与公共图片支持集合一致 + +### Requirement: Distinct input failure reasons + +系统 SHALL 区分缺失路径、不支持格式、目录无有效帧和解码失败；提示 SHALL 适用于 Image Empty 和 Image Texture，解码异常转换 SHALL 限定于读取边界。 + +#### Scenario: Missing source file +- **WHEN** Image Empty 或材质 Image Texture 的源图片或视频文件不存在 +- **THEN** 提示缺失的 source image/video file 及路径，不将原因报告为格式不支持 + +#### Scenario: Invalid encoded data +- **WHEN** 支持扩展名的文件内容无法作为对应图片或视频解码 +- **THEN** 报告无法读取的媒体类型及路径，并保留底层异常供诊断 + +### Requirement: Explicit frame selection + +系统 SHALL 在复制 Blender 序列前校验图片格式；通用目录 SHALL 仅收集支持的图片，保持现有排序及帧范围规则。 + +#### Scenario: Unsupported Blender sequence +- **WHEN** Blender 源序列使用 `.exr` 扩展名 +- **THEN** 复制前提示不支持的图片格式，不创建临时帧目录 + +#### Scenario: Mixed directory +- **WHEN** 通用输入目录包含 JPG、PNG、TXT 和 EXR 文件 +- **THEN** 仅 JPG 与 PNG 参与处理，前后端按相同规则选择并保持原帧范围语义 + +#### Scenario: Empty supported frame set +- **WHEN** 通用目录为空或只有不支持的文件 +- **THEN** 提示无支持的图片帧、目录路径及支持的图片格式 + +### Requirement: Validate prepared media + +系统 SHALL 校验实际提交的文件，保持各功能自身的媒体类型、尺寸和比例约束。 + +#### Scenario: Packed image exported as PNG +- **WHEN** 一个名称带 `.exr` 的 packed Image 经现有流程成功导出 PNG +- **THEN** 按导出的 PNG 校验，不按源名称拒绝 + +#### Scenario: Feature restriction +- **WHEN** 输入通过公共格式校验但不满足 Depth Plane 的静态图限制或 Upscale 的尺寸限制 +- **THEN** 仍按对应业务原因取消，不报告为格式不支持 diff --git a/openspec/changes/unify-ai-input-validation/specs/ai-operator-failure-handling/spec.md b/openspec/changes/unify-ai-input-validation/specs/ai-operator-failure-handling/spec.md new file mode 100644 index 0000000..35a2d3a --- /dev/null +++ b/openspec/changes/unify-ai-input-validation/specs/ai-operator-failure-handling/spec.md @@ -0,0 +1,33 @@ +## ADDED Requirements + +### Requirement: Expected operator failures are reported normally + +AI 操作 SHALL 将输入准备与嵌套 Operator 调用的预期失败转换为普通错误提示和 CANCELLED，包括 Remove BG、Depth/Relief Plane、AI Environment 准备及现有 Panorama 入口。提示 SHALL 保留原因并避免重复的 Blender 错误前缀。 + +#### Scenario: Nested input validation failure +- **WHEN** 内部 Job Operator 报告无效输入且 bpy.ops 抛出 RuntimeError +- **THEN** 外部入口返回 CANCELLED，显示原因，不向用户展示未捕获的 Python traceback + +#### Scenario: Environment setup failure +- **WHEN** AI Environment 设置入口调用安装或下载 Operator 同步失败 +- **THEN** 设置入口及调用它的 AI 操作按普通错误取消 + +#### Scenario: Successful job startup +- **WHEN** 内部 Job Operator 返回 RUNNING_MODAL +- **THEN** 外部入口保持现有成功返回与异步处理语义，源图片及 Undo 行为保持有效 + +### Requirement: Temporary inputs have explicit ownership + +系统 SHALL 释放本次操作创建的临时输入，在准备失败、启动失败、取消及异步结束时均有负责清理的边界；用户原文件 SHALL 保持不变。 + +#### Scenario: Preparation failure +- **WHEN** packed 图片导出或序列复制失败 +- **THEN** 准备函数删除本次临时文件和目录，并保留源图片与源文件 + +#### Scenario: Synchronous nested failure +- **WHEN** 临时输入准备成功后内部 Operator 抛错或返回 CANCELLED +- **THEN** 临时输入被清理，外部操作取消，源图片引用保持不变 + +#### Scenario: Asynchronous completion +- **WHEN** Job 接管临时输入并在之后成功、失败或取消 +- **THEN** 输入在 Job 结束时清理，运行期间保持可用，用户原文件不被删除 diff --git a/openspec/changes/unify-ai-input-validation/tasks.md b/openspec/changes/unify-ai-input-validation/tasks.md new file mode 100644 index 0000000..88492fd --- /dev/null +++ b/openspec/changes/unify-ai-input-validation/tasks.md @@ -0,0 +1,30 @@ +## 1. 公共媒体校验 + +- [x] 1.1 在 server/media/input.py 统一路径、图片/视频扩展名与格式消息，保持标准库预检及后端独立加载；补充大小写、无扩展名和缺失路径测试。 +- [x] 1.2 让 prepare_input 与 BEN2 单帧校验复用公共规则；验证直接后端请求与前端提示一致，图片子集限制有效。 +- [x] 1.3 在实际图片/视频读取边界统一解码失败提示并保留异常原因；覆盖损坏内容，确认推理异常保留自身语义。 + +## 2. Blender 输入准备 + +- [x] 2.1 将 require_input_path 接入公共预检，移除 require_background_input_path 和 BEN2 重复常量；覆盖 Remove BG/Upscale 同一无效输入的消息及未提交 Job 行为。 +- [x] 2.2 统一 source image/video 路径提示，覆盖 Image Empty 和材质 Image Texture；回归其他公共校验消费者。 +- [x] 2.3 在 Blender 序列复制前校验图片格式；验证不支持序列、混合目录、无有效帧目录、帧排序与范围。 +- [x] 2.4 修复输入准备失败的临时目录释放，验证 packed 导出与序列复制失败；覆盖普通原文件和导出 PNG 的不同实际输入路径。 + +## 3. Operator 失败边界 + +- [x] 3.1 修复 Remove BG、Depth/Relief Plane 的嵌套调用异常出口，规范错误前缀并处理同步失败/CANCELLED 的临时输入清理。 +- [x] 3.2 补齐公共 AI setup 入口及 AI Environment 安装/下载嵌套调用的预期失败处理；回归 Panorama 已有错误边界。 +- [x] 3.3 补充准备失败、同步取消、异步成功/失败/取消的生命周期测试，确认原文件、源图片引用和 Undo 行为保持有效。 + +## 4. 验证 + +- [x] 4.1 运行相关测试，再运行 uv run pytest 并确认正常退出；检查旧校验名称、重复格式列表与最终差异。 +- [x] 4.2 在真实 Blender UI 从 Remove BG、Depth/Relief Plane 及 AI setup 入口触发同步失败，确认普通提示无重复前缀、无未捕获 traceback，并检查临时输入已释放。 +- [x] 4.3 验证有效 PNG、视频、序列与材质贴图入口，记录测试结果及实际验证限制。 + +## 验证记录 + +- `uv run pytest -q`：1025 passed、2 xfailed、52 subtests passed，退出码 0。随后新增的 4 项原生 Operator 测试通过；最终相关新增测试合计 24 项通过。 +- Blender 4.5.10 LTS 独立场景：UI 主循环依次调用 Remove BG、Upscale、Depth、Relief 和 AI setup；前四者格式错误完全一致，所有入口仅一层 Error 前缀且没有未捕获 traceback。原生回归另行验证同步失败后的临时输入清理。 +- 有效图片、视频帧、序列、材质贴图、异步完成/取消/失败和 Undo 由全量回归覆盖；本次验证使用模拟推理与安装失败，不下载模型或执行真实模型推理。 diff --git a/openspec/changes/unify-material-color-images/.openspec.yaml b/openspec/changes/unify-material-color-images/.openspec.yaml new file mode 100644 index 0000000..e8cda9e --- /dev/null +++ b/openspec/changes/unify-material-color-images/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-10 diff --git a/openspec/changes/unify-material-color-images/design.md b/openspec/changes/unify-material-color-images/design.md new file mode 100644 index 0000000..7d827b1 --- /dev/null +++ b/openspec/changes/unify-material-color-images/design.md @@ -0,0 +1,71 @@ +## Context + +动机与范围见 proposal.md。普通 Cutout 在内存中新建颜色图，AI Cutout 从裁切临时文件加载；Convert To 使用按 Empty 引用决定复用的分支。剪贴板通过输入字节哈希查找 packed Image，编辑覆盖原 Image 后该标记仍然存在。 + +## Goals / Non-Goals + +设计目标是让所有材质入口共享范围采样和颜色图交付逻辑，并让剪贴板按当前图像状态验证缓存。输入范围统一使用现有像素坐标约定，静态颜色图以独立 packed Image 交付。 + +## Decisions + +### 1. 统一范围与颜色文件准备 + +```mermaid +flowchart TD + A[读取当前源图像素和解释设置] --> B[确定像素范围] + B --> C[Cutout 使用裁切范围] + B --> D[Convert To 和剪贴板 Plane 使用整图范围] + C --> E[导出唯一临时颜色文件] + D --> E + E --> F[独立加载并命名为原名_color] + F --> G[配置材质颜色空间和 Alpha] + G --> H[打包并绑定材质] + E --> I[需要 AI 时准备模型输入并生成几何数据] + H --> J[完成对象交付并清理临时资源] + I --> J +``` + +公共准备逻辑放在 common，输入源 Image、当前像素及范围。复用现有像素读取、路径清理和 packed 加载能力；由临时产物携带采样时的尺寸与解释设置，异步响应使用同一次准备的内容。AI 和普通 Cutout 采用相同裁切颜色来源，模型推理的缩放或预览处理只作用于分析输入。 + +### 2. 独立加载是材质所有权边界 + +每次材质创建都从本次临时颜色文件使用 `check_existing=False` 加载新的 Color Image。名称使用源图基名、`_color` 和实际编码后缀，由 Blender 处理重名。文件位于本次操作的唯一临时目录，与原图路径分离。 + +材质构建函数接收已准备好的独立 Color Image，负责配置和节点绑定。所有入口在进入该函数前完成相同准备；Cutout 已准备的产物直接交付。原 Image 的像素、名称、路径、颜色空间和 Alpha 保持原值。材质 Image 由文件独立加载，因此不继承源 Image 的剪贴板缓存属性。 + +### 3. 编码精度与颜色解释共同验证 + +普通 byte 图片使用无损 PNG；float / HDR 使用保留浮点范围和精度的 EXR。复用 common 中适合的 HDR 编码能力，按源 Alpha 解释正确处理关联 RGB 和透明区域颜色。 + +普通 byte 材质沿用适配偏好、inverse 空间映射与 sRGB 回退。所有 float / HDR 材质保留源颜色空间与 Alpha 模式；Panorama 保留现有非 sRGB 规则。导出、独立加载、配置、打包和保存重载作为完整往返验证。普通 float 不再强制套用显示颜色空间，原有重载 xfail 改为源解释保留的正常回归测试。 + +float / HDR 在导出后按源解释独立加载并比较像素。STRAIGHT 按 Alpha 解关联后编码；源解释与 EXR 无法同时保留像素时明确拒绝转换并清理临时目录，例如 STRAIGHT 下完全透明像素仍含非零 RGB。float 材质不启用场景视图适配的 Alpha Fix。 + +### 4. 临时产物由操作持有 + +同步操作持有临时目录直至 Color 完成打包和对象交付；异步操作持有颜色产物及分析输入直至响应、取消或失败处理完成。静态普通颜色文件适合作为模型输入时复用同一临时文件；HDR 分析预览保留独立路径。所有清理入口按真实持有关系释放资源，packed Color 在临时文件删除后仍可保存和重载。 + +转换源对象与 Image 身份校验继续使用；材质隔离后清理原图复用及相应的原图配置恢复、打包撤销基线分支。保留对象、材质、网格和临时 Image 的异常清理及转换 Undo / Redo。 + +### 5. 剪贴板缓存验证实际内容 + +输入字节的 SHA256 继续作为候选索引。首次加载 packed Image 后保存其内容指纹：尺寸、统一读取的 RGBA 像素、颜色空间及 Alpha 模式。命中输入哈希时重新计算当前指纹，匹配才复用；不匹配、缺少指纹或无法读取时跳过该候选，从剪贴板原始字节重新创建。 + +指纹在稳定的像素读取约定下计算，包含完全透明像素的 RGB。使用前验证能同时覆盖 Mask、Frame、Rectify、AI 编辑与原生绘制，避免依赖每个编辑入口主动通知。`is_dirty` 不能代表内容是否仍相同，packed 的编辑结果也必须验证。 + +剪贴板粘贴为参考图、节点或笔刷时沿用有效原图缓存；粘贴为 Plane 材质时基于该原图按整图范围创建独立颜色产物。旧缓存缺少指纹时按未命中处理，无需迁移用户图像。 + +### 6. Plane 收紧为静态图片 + +在创建材质或临时文件前拒绝 Movie、Sequence 及其他动画输入，明确报告仅支持静态图。移除 Plane 的帧起点、偏移、时长及循环传递，并沿调用链清理仅供材质动画使用的参数。其他图片编辑功能仍按各自输入约定运行。 + +## Risks / Trade-offs + +- [PNG / EXR 往返改变 RGB 或 Alpha 解释] → 覆盖 byte、float、HDR、透明 RGB、dirty、generated 和 packed 来源，检查配置前后与保存重载结果。 +- [全图内容指纹存在读取成本] → 先以剪贴板输入哈希筛选，仅验证匹配候选，复用 common 的无副作用像素读取。 +- [临时文件被提前删除] → 操作持有文件直至模型输入使用结束与材质打包完成，测试取消和失败路径。 +- [旧复用测试与当前要求冲突] → 以本提案的独立 Color 规格更新行为断言，保留有效错误恢复覆盖。 + +## Migration Plan + +先实现并验证公共颜色产物，再接入 Cutout、Convert To 与剪贴板 Plane；补充缓存指纹校验，移除被替换的原图复用分支和重复实现。仅对后续操作采用新规则，已有场景数据保留。更新相关测试并运行全量测试，确认旧引用清理和正常退出。 diff --git a/openspec/changes/unify-material-color-images/proposal.md b/openspec/changes/unify-material-color-images/proposal.md new file mode 100644 index 0000000..5ba76ec --- /dev/null +++ b/openspec/changes/unify-material-color-images/proposal.md @@ -0,0 +1,28 @@ +## Why + +材质直接复用原 Image 会让再次导入原文件或再次粘贴剪贴板内容带入材质配置及绘制结果。Cutout 与 Convert To 需要统一的独立颜色图流程，剪贴板缓存则需要确认当前图像仍代表最初粘贴的内容。 + +## What Changes + +- Cutout、Plane、Depth Plane、Relief Plane、Panorama 和剪贴板 Plane 统一按图片范围生成材质颜色图：导出独立临时文件、独立加载、以 `_color` 命名、配置并打包。 +- Cutout 始终使用裁切范围；Convert To 和剪贴板 Plane 使用整图范围。是否使用 AI 只影响深度、法线等推理产物。 +- 材质颜色图与原 Image、原文件加载路径及剪贴板缓存标记隔离，保留源像素精度和各入口的颜色空间、Alpha 规则。 +- 普通 byte 图继续场景视图适配；所有 float / HDR 保留源解释，无法无损往返的输入明确拒绝转换。 +- 保留相同剪贴板内容的复用；缓存命中时核对当前像素、尺寸、颜色空间和 Alpha 模式，修改后的图片不再代表原缓存内容。 +- **BREAKING**：Convert to Plane 仅接受静态图片，拒绝 Movie 和 Sequence，移除该入口的动画帧参数传递。 +- 保留失败清理、异步源图校验与撤销能力，清理转换原图复用分支和重复颜色图实现。 + +## Capabilities + +### New Capabilities + +- `material-color-images`: 所有材质创建入口的范围采样、独立颜色图、精度、配置、静态输入和资源生命周期。 +- `clipboard-image-cache`: 相同剪贴板内容的有效复用、编辑后的缓存失效以及材质派生图隔离。 + +### Modified Capabilities + +## Impact + +涉及 `common/image.py`、`common/material.py`、Cutout、两个 Convert To 包、剪贴板图像获取及 Plane 创建入口和相关测试。沿用现有依赖与节点资产。 + +本提案取代 `reuse-conversion-source-images` 的材质原图复用策略及 Plane 动画约定；采用当前代码中的失败清理和源对象校验能力继续实施。当前主规格目录为空，新规格集中表达最终行为，后续归档时以本提案的重叠约定为准。 diff --git a/openspec/changes/unify-material-color-images/specs/clipboard-image-cache/spec.md b/openspec/changes/unify-material-color-images/specs/clipboard-image-cache/spec.md new file mode 100644 index 0000000..5992bfc --- /dev/null +++ b/openspec/changes/unify-material-color-images/specs/clipboard-image-cache/spec.md @@ -0,0 +1,43 @@ +## Purpose + +保留相同剪贴板图片的复用能力，同时核对候选 Image 当前内容与解释设置是否仍匹配首次加载状态，确保编辑后的图像和材质派生图不会被误认为原始剪贴板内容再次粘贴。 + +## ADDED Requirements + +### Requirement: Clipboard reuse requires an unchanged image + +相同剪贴板输入 SHALL 仅复用当前像素、尺寸、颜色空间和 Alpha 模式均匹配缓存基线的有效 Image。缓存验证 SHALL 包括透明区域 RGB，并保持候选图像原有内容与状态。 + +#### Scenario: Repeated content remains unchanged + +- **WHEN** 相同剪贴板内容再次粘贴，缓存图像内容和设置未改变 +- **THEN** 系统 SHALL 复用该 Image + +#### Scenario: Mask overwrites the cached image + +- **WHEN** Mask 修改了独占的剪贴板 Image，修改结果已打包，再次粘贴原始剪贴板内容 +- **THEN** 系统 SHALL 加载原始剪贴板内容到另一 Image，保留已编辑图像 + +#### Scenario: Shared edit leaves an intact candidate + +- **WHEN** 编辑产生独立结果，另一个用户仍引用未修改的剪贴板原图 +- **THEN** 再次粘贴 SHALL 复用该未修改原图 + +#### Scenario: Other editing or interpretation changes invalidate a candidate + +- **WHEN** 候选图像经过尺寸修改、原生绘制、透明 RGB 修改、颜色空间或 Alpha 模式变更 +- **THEN** 再次粘贴 SHALL 跳过已变更候选，使用原始剪贴板内容及其初始解释设置 + +#### Scenario: Legacy or unreadable cache entry is found + +- **WHEN** 候选缺少内容校验基线或无法读取 +- **THEN** 系统 SHALL 按未命中处理，并保留已有图像 + +### Requirement: Clipboard material results stay outside the source cache + +剪贴板 Plane SHALL 从有效剪贴板原图按整图范围生成独立材质颜色图。参考图、节点和笔刷的后续粘贴 SHALL 按原图缓存有效性选择 Image。 + +#### Scenario: Painted material is followed by another paste + +- **WHEN** 用户粘贴为 Plane 并绘制其材质，随后把相同内容粘贴为参考图、节点或笔刷 +- **THEN** 系统 SHALL 使用原始剪贴板内容，且不复用材质颜色图 diff --git a/openspec/changes/unify-material-color-images/specs/material-color-images/spec.md b/openspec/changes/unify-material-color-images/specs/material-color-images/spec.md new file mode 100644 index 0000000..c332349 --- /dev/null +++ b/openspec/changes/unify-material-color-images/specs/material-color-images/spec.md @@ -0,0 +1,82 @@ +## Purpose + +统一 Cutout、Convert To 与剪贴板 Plane 的材质颜色图来源和交付约定，使裁切与整图转换使用同一范围采样流程，并保障材质编辑、再次导入原文件及保存重载时图像内容与设置保持隔离。 + +## ADDED Requirements + +### Requirement: Material color uses the requested source bounds + +所有材质生成入口 SHALL 按本次准备时的源图像素和范围生成颜色图。Cutout SHALL 使用裁切范围，四种 Convert To 与剪贴板 Plane SHALL 使用整图范围。AI 的参与 SHALL 保持相同颜色来源及颜色分辨率。 + +#### Scenario: Cutout uses or omits AI + +- **WHEN** 同一源图和裁切范围分别执行普通 Cutout 和需要 AI 的 Cutout +- **THEN** 两者的颜色图 SHALL 对应同一裁切内容、尺寸和原有 Alpha，包含透明区域 RGB + +#### Scenario: A full image is converted + +- **WHEN** 执行 Plane、Depth Plane、Relief Plane、Panorama 或剪贴板 Plane +- **THEN** 颜色图 SHALL 对应完整源图片，保持原始尺寸和当前像素 + +### Requirement: Every material owns an isolated color image + +每次生成材质 SHALL 使用独立 Color Image 和独立临时颜色文件路径，以源图基名、`_color` 及实际文件后缀命名并打包。原 Image 的内容和解释设置 SHALL 保持不变，材质 Color SHALL 不携带剪贴板原图缓存身份。 + +#### Scenario: Source users vary + +- **WHEN** 原图没有其他用户，或被其他 Empty、材质或笔刷引用 +- **THEN** 新材质 SHALL 使用独立颜色图，原图的引用、像素、名称、路径、颜色空间及 Alpha 模式 SHALL 保持原值 + +#### Scenario: Original file is imported after material painting + +- **WHEN** 转换后修改材质颜色图，再次按原文件路径导入图片，包括原 Image 已移除的情况 +- **THEN** 导入 SHALL 不命中材质颜色图，不带入该颜色图的材质设置和绘制像素 + +### Requirement: Encoding preserves color precision and presentation + +颜色图 SHALL 保留对应范围的 RGB、Alpha、透明区域 RGB 及源编码精度，float / HDR SHALL 保留浮点范围。普通 byte 材质 SHALL 沿用颜色空间适配与 Alpha 配对规则，所有 float / HDR 材质 SHALL 保留源颜色空间与 Alpha 模式。 + +#### Scenario: Ordinary material is configured + +- **WHEN** 普通 byte 材质适配关闭或回退到 sRGB，或者启用可用的 inverse 空间 +- **THEN** 颜色图 SHALL 分别采用 sRGB + PREMUL 或对应 inverse 空间 + STRAIGHT，原图设置保持不变 + +#### Scenario: HDR and unsaved pixels are serialized + +- **WHEN** float / HDR、generated、dirty 或 packed 源图完成颜色图生成、打包和保存重载 +- **THEN** 结果 SHALL 在对应编码精度内保留业务像素、动态范围与选定的颜色解释 +- **AND** 所有 float / HDR 材质 SHALL 保留源图颜色空间与 Alpha 模式；Panorama 非 sRGB 分支沿用源解释 + +#### Scenario: Float source interpretation prevents lossless serialization + +- **WHEN** 按源颜色空间和 Alpha 模式解码 EXR 后无法保留源浮点像素 +- **THEN** 操作 SHALL 明确拒绝转换，清理临时资源并保持源图不变 +- **AND** float / HDR 材质 SHALL 不启用场景视图适配的 Alpha Fix + +### Requirement: Material operations release temporary resources safely + +临时颜色文件 SHALL 在打包和相关 Job 使用结束后释放。取消或失败 SHALL 清理本次临时文件与未交付数据，保留源对象及原图。成功转换 SHALL 支持 Undo / Redo；异步源对象失效或 Image 被替换时 SHALL 取消应用。 + +#### Scenario: Packed result survives file cleanup + +- **WHEN** 操作成功且临时目录已删除 +- **THEN** 材质颜色图 SHALL 仍能正确读取并保存重载 + +#### Scenario: An operation fails or is cancelled + +- **WHEN** 导出、模型处理、图像加载、材质或对象创建失败，或用户取消 Job +- **THEN** 本次临时资源 SHALL 释放，源对象、源像素和源图设置保持原值 + +#### Scenario: Conversion is undone and redone + +- **WHEN** 用户撤销并重做转换 +- **THEN** 撤销 SHALL 恢复源对象和源图，重做 SHALL 恢复带独立颜色图的转换结果 + +### Requirement: Plane accepts static images only + +Convert to Plane SHALL 仅接受静态图片；Movie、Sequence 和动画图像 SHALL 在创建颜色图或对象前被拒绝，并给出明确错误。 + +#### Scenario: Animated source is selected + +- **WHEN** 源 Image 为 Movie 或 Sequence，即使其声明时长为一帧，或源图片含多帧 +- **THEN** Plane SHALL 报告仅支持静态图片，并保持源对象及图像不变 diff --git a/openspec/changes/unify-material-color-images/tasks.md b/openspec/changes/unify-material-color-images/tasks.md new file mode 100644 index 0000000..f217044 --- /dev/null +++ b/openspec/changes/unify-material-color-images/tasks.md @@ -0,0 +1,25 @@ +## 1. 公共颜色图流程 + +- [x] 1.1 在 common 中统一按范围导出、独立加载、命名与打包的颜色产物流程，明确临时目录所有权；测试整图和局部裁切的尺寸、RGB、Alpha、独立 Image 身份及原路径隔离。 +- [x] 1.2 实现 byte PNG 和 float / HDR EXR 的精度与解释配对，复用适合的公共像素和编码能力；验证 generated、dirty、packed、透明 RGB、HDR 动态范围及材质配置后的保存重载，保留 float / HDR 源解释，无法无损往返时明确拒绝，替换原普通 float 重载 xfail。 +- [x] 1.3 保留公共材质函数对已准备 Color 的配置及节点绑定职责，验证 inverse / sRGB 回退、Alpha 配对、Panorama HDR 保留和源 Image 设置不变。 + +## 2. 接入材质入口 + +- [x] 2.1 将普通与 AI Cutout 接入同一裁切颜色产物，协调分析输入与异步文件生命周期；同源同范围测试确认两条路径颜色一致、透明 RGB 保留，推理分辨率不改变颜色分辨率。 +- [x] 2.2 将 Plane、Depth Plane、Relief Plane 和 Panorama 接入整图范围流程，移除转换原图复用及相应恢复分支，保留异步源身份校验；验证不同共享用户下原图不变、四个入口使用独立 `_color` Image。 +- [x] 2.3 将 Plane 限定为静态图片并清理该入口帧参数及仅供材质动画使用的路径；测试 Movie、Sequence 单帧与多帧输入在创建临时资源前被拒绝，静态转换与 Operator 注册正常。 +- [x] 2.4 将剪贴板 Plane 接入整图颜色产物，确保结果不携带缓存身份；测试创建材质后绘制再粘贴为参考图、节点或笔刷，得到原始剪贴板内容。 + +## 3. 剪贴板缓存有效性 + +- [x] 3.1 为新缓存建立内容指纹，按输入哈希筛选后核对当前像素、尺寸、颜色空间与 Alpha；测试未变内容复用、变化内容跳过、透明 RGB 变化、旧缓存缺少指纹和候选读取失败，验证读取不改变候选状态。 +- [x] 3.2 补充实际编辑后的重复粘贴测试，覆盖 Mask 覆盖原 Image、已打包编辑结果、共享 Empty 保留原图、尺寸修改及原生绘制；确认原始粘贴内容正确且编辑结果保留。 + +## 4. 生命周期与最终验证 + +- [x] 4.1 验证导出、加载、模型、材质和对象创建失败以及 Job 取消的资源清理，确认原图与源对象保持有效；通过临时目录删除后的保存重载及转换 Undo / Redo 测试。 +- [x] 4.2 添加“生成材质 → 绘制 → 再按原文件路径导入”的回归，包含原 Image 已删除场景，验证不会命中材质 Color；检查所有材质入口及重复实现的最终引用。 +- [x] 4.3 更新受影响测试并运行全量 `uv run pytest`，确认正常退出；检查最终差异、旧引用和 OpenSpec 严格校验，保留工作区已有修改。 + +验证结果：`uv run pytest` 共 1162 项通过，退出码 0；OpenSpec 严格校验通过。 diff --git a/src/anyimage/assets/O_AnyImage.blend b/src/anyimage/assets/O_AnyImage.blend index 25181d3..e73353b 100644 --- a/src/anyimage/assets/O_AnyImage.blend +++ b/src/anyimage/assets/O_AnyImage.blend @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c3e6c8cffd8c69efb3d4770feccd6d9a6d2c8bf23d838b45b4bf75473c21634d -size 6146627 +oid sha256:5b6c752747d88f09879401ed6a723a03077d6ac33804ae2695b8e5feefdfe7ef +size 7441494 diff --git a/src/anyimage/common/ai.py b/src/anyimage/common/ai.py index 1047e93..5b98559 100644 --- a/src/anyimage/common/ai.py +++ b/src/anyimage/common/ai.py @@ -1,9 +1,9 @@ """Shared Blender-side AI setup, validation, UI, and Job parameters.""" -from pathlib import Path - import bpy +from ..server.media.input import validate_input_path + from ..preferences import addon_preferences from ..properties import ( ai_ready, @@ -14,11 +14,6 @@ from ..properties import ( ) from ..runtime import runtime -BEN2_IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp"} -BEN2_VIDEO_SUFFIXES = { - ".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".mts", ".mpg", ".mpeg" -} - def draw_ai_setup(layout): status = ai_status() @@ -38,21 +33,12 @@ def draw_ai_property(layout, owner, name, status): def require_input_path(value): - path = Path(bpy.path.abspath(value)) - if path.is_file() or path.is_dir(): - return path - raise ValueError("Select a valid image, video, or frame directory") + return validate_input_path(bpy.path.abspath(value)) -def require_background_input_path(value): - path = require_input_path(value) - if path.is_dir() or path.suffix.lower() in ( - BEN2_IMAGE_SUFFIXES | BEN2_VIDEO_SUFFIXES - ): - return path - raise ValueError( - "BEN2 requires an image, video, or directory of image frames" - ) +def report_ai_error(operator, error): + """Report the underlying reason from a nested Blender operator.""" + operator.report({"ERROR"}, str(error).removeprefix("Error: ").strip()) def production_device(): @@ -64,10 +50,14 @@ def require_environment(): raise RuntimeError("Install the AnyImage Environment first") -def invoke_ai_setup_if_needed(): +def invoke_ai_setup_if_needed(operator): if ai_ready(): return None - return bpy.ops.anyimage.setup_ai_environment("INVOKE_DEFAULT") + try: + return bpy.ops.anyimage.setup_ai_environment("INVOKE_DEFAULT") + except RuntimeError as error: + report_ai_error(operator, error) + return {"CANCELLED"} def require_model(model_key): diff --git a/src/anyimage/common/color_image.py b/src/anyimage/common/color_image.py new file mode 100644 index 0000000..ca807b8 --- /dev/null +++ b/src/anyimage/common/color_image.py @@ -0,0 +1,139 @@ +"""Prepare isolated material color files from static image bounds.""" + +import json +import tempfile +from contextlib import contextmanager +from pathlib import Path + +import bpy + +from .image import cleanup_image_input, image_base_name, image_is_packed, image_rgba, is_animated_image, video_total_frames + + +def require_static_color_image(image): + """Reject animated inputs before preparing material resources.""" + if is_animated_image(image) or video_total_frames(image) != 1: + raise ValueError("Material conversion supports static images only") + + +def _save_byte_color(rgba, path, color_space): + import numpy as np + + height, width = rgba.shape[:2] + image = bpy.data.images.new("Material Color Export", width=width, height=height, alpha=True) + try: + image.colorspace_settings.name = color_space + image.alpha_mode = "STRAIGHT" + image.pixels.foreach_set(np.flipud(rgba).ravel()) + image.file_format = "PNG" + image.save(filepath=str(path), save_copy=True) + if not path.is_file(): + raise RuntimeError("Unable to export the material color image") + finally: + bpy.data.images.remove(image) + + +def prepare_material_color_input(source, bounds=None, rgba=None): + """Export source bounds with interpretation matching the encoded color file.""" + import numpy as np + from .hdr_image import save_hdr_result + + require_static_color_image(source) + width, height = map(int, source.size) + if (source.is_float and source.source == "FILE" and not source.is_dirty + and not image_is_packed(source) and source.alpha_mode == "STRAIGHT"): + # Read the encoded float colors without applying a second alpha multiplication. + reader = source.copy() + try: + reader.alpha_mode = "PREMUL" + rgba = image_rgba(reader) + finally: + bpy.data.images.remove(reader) + else: + rgba = image_rgba(source) if rgba is None else np.asarray(rgba, dtype=np.float32) + if rgba.shape != (height, width, 4): + raise ValueError("The source RGBA does not match the Blender Image") + left, top, right, bottom = (0, 0, width, height) if bounds is None else map(int, bounds) + if not (0 <= left < right <= width and 0 <= top < bottom <= height): + raise ValueError("The material color bounds are outside the image") + cropped = rgba[top:bottom, left:right] + suffix = ".exr" if source.is_float else ".png" + path = Path(tempfile.mkdtemp(prefix="anyimage-color-")) / ("color" + suffix) + try: + if source.is_float: + # Store float colors directly; the EXR interpretation is applied on load. + save_hdr_result(cropped, path) + else: + _save_byte_color(cropped, path, source.colorspace_settings.name) + path.with_suffix(".json").write_text(json.dumps({ + "name": image_base_name(source) + "_color" + suffix, + "color_space": "Linear Rec.709" if source.is_float else source.colorspace_settings.name, + "alpha_mode": "PREMUL" if source.is_float and source.alpha_mode == "STRAIGHT" else source.alpha_mode, + }), encoding="utf-8") + return path + except Exception: + cleanup_material_color_input(path) + raise + + +def material_analysis_input(color_path): + """Return the byte color file or an HDR recognition preview in the same directory.""" + from .hdr_image import recognition_rgba + + path = Path(color_path) + if path.suffix == ".png": + return path + image = load_material_color_image(path) + try: + rgba = image_rgba(image) + if image.alpha_mode == "NONE": + rgba[..., 3] = 1 + preview = recognition_rgba(rgba, associated=image.alpha_mode not in {"CHANNEL_PACKED", "NONE"}) + output = path.parent / "analysis.png" + _save_byte_color(preview, output, "sRGB") + return output + finally: + bpy.data.images.remove(image) + + +def cleanup_material_color_input(path): + """Release the directory owned by a prepared material color input.""" + if path: + cleanup_image_input(Path(path).parent, True) + + +def load_material_color_image(path): + """Load an independent packed Color image using its file interpretation.""" + path = Path(path) + metadata = json.loads(path.with_suffix(".json").read_text(encoding="utf-8")) + image = bpy.data.images.load(str(path), check_existing=False) + try: + image.name = metadata["name"] + image.colorspace_settings.name = metadata["color_space"] + image.alpha_mode = metadata["alpha_mode"] + image.pack() + return image + except Exception: + bpy.data.images.remove(image) + raise + + +@contextmanager +def material_color_image(source=None, *, bounds=None, rgba=None, input_path=None): + """Own a prepared color image through material creation and clean failed results.""" + owned = input_path is None + path = prepare_material_color_input(source, bounds, rgba) if owned else Path(input_path) + image = None + try: + image = load_material_color_image(path) + yield image + except Exception: + if image is not None: + bpy.data.images.remove(image, do_unlink=True) + image = None + raise + finally: + if image is not None and image.users == 0: + bpy.data.images.remove(image) + if owned: + cleanup_material_color_input(path) diff --git a/src/anyimage/common/hdr_image.py b/src/anyimage/common/hdr_image.py new file mode 100644 index 0000000..d98e15c --- /dev/null +++ b/src/anyimage/common/hdr_image.py @@ -0,0 +1,177 @@ +"""Prepare HDR recognition previews and commit float background-removal results.""" + +from dataclasses import dataclass +from pathlib import Path +import tempfile + +import bpy +import numpy as np + +from .image import ( + cleanup_image_input, image_content_state, image_rgba, is_animated_image, + restore_image_content, +) +from .image_target import has_other_image_user + + +def recognition_rgba(rgba, *, associated=True): + """Tone-map scene-linear pixels for sRGB recognition without changing the source.""" + alpha = rgba[..., 3:4] + rgb = rgba[..., :3].astype(np.float64) + if associated: + rgb = np.divide(rgb, alpha, out=np.zeros_like(rgb), where=alpha > 0) + rgb = np.maximum(rgb, 0) + luminance = rgb @ np.array([0.2126, 0.7152, 0.0722]) + rgb /= 1 + luminance[..., None] + encoded = np.where(rgb <= 0.0031308, rgb * 12.92, 1.055 * rgb ** (1 / 2.4) - 0.055) + preview = np.ones(rgba.shape, dtype=np.float32) + preview[..., :3] = np.clip(encoded, 0, 1) * alpha + return preview + + +def save_hdr_result(rgba, path): + """Save associated scene-linear RGBA as ZIP-compressed 32-bit EXR.""" + height, width = rgba.shape[:2] + image = bpy.data.images.new("AnyImage HDR Result", width=width, height=height, + alpha=True, float_buffer=True) + scene = None + try: + image.alpha_mode = "PREMUL" + image.colorspace_settings.name = "Linear Rec.709" + image.pixels.foreach_set(np.flipud(rgba).ravel()) + scene = bpy.data.scenes.new("AnyImage HDR Export") + settings = scene.render.image_settings + settings.file_format = "OPEN_EXR" + settings.color_mode = "RGBA" + settings.color_depth = "32" + settings.exr_codec = "ZIP" + image.save_render(str(path), scene=scene) + finally: + if scene is not None: + bpy.data.scenes.remove(scene) + bpy.data.images.remove(image) + + +def create_packed_hdr(rgba, name, path, *, alpha_mode="PREMUL"): + """Create a self-contained float image and verify its encoded pixels.""" + result = None + save_hdr_result(rgba, path) + try: + result = bpy.data.images.load(str(path), check_existing=False) + result.alpha_mode = alpha_mode + if (not result.is_float + or result.colorspace_settings.name != "Linear Rec.709" + or not np.allclose(image_rgba(result), rgba, rtol=2e-6, atol=1e-7)): + raise RuntimeError("The HDR result did not preserve float colors") + result.pack() + result.name = name + return result + except Exception: + if result is not None: + bpy.data.images.remove(result) + raise + + +@dataclass +class HdrBackgroundInput: + """Own the submitted pixels and color interpretation of an HDR still.""" + + rgba: np.ndarray + color_space: str + alpha_mode: str + source: str + path: Path + + @classmethod + def prepare(cls, image): + if is_animated_image(image): + raise RuntimeError("HDR background removal supports still images only") + if image.source not in {"FILE", "GENERATED"}: + raise RuntimeError("HDR background removal requires a file or generated image") + if image.colorspace_settings.is_data: + raise RuntimeError("HDR background removal requires a color image with opacity alpha") + rgba = image_rgba(image).copy() + if not np.isfinite(rgba).all() or np.any((rgba[..., 3] < 0) | (rgba[..., 3] > 1)): + raise RuntimeError("HDR image contains invalid color or alpha values") + path = Path(tempfile.mkdtemp(prefix="anyimage-hdr-")) / "input.png" + preview = None + try: + height, width = rgba.shape[:2] + preview = bpy.data.images.new("AnyImage HDR Preview", width=width, height=height, + alpha=True) + preview.colorspace_settings.name = "sRGB" + preview.alpha_mode = "STRAIGHT" + preview_source = rgba.copy() + if image.alpha_mode == "NONE": + preview_source[..., 3] = 1 + preview_rgba = recognition_rgba( + preview_source, associated=image.alpha_mode not in {"CHANNEL_PACKED", "NONE"}, + ) + preview.pixels.foreach_set(np.flipud(preview_rgba).ravel()) + preview.file_format = "PNG" + preview.save(filepath=str(path), save_copy=True) + if not path.is_file(): + raise RuntimeError("Blender did not export the HDR recognition preview") + except Exception: + cleanup_image_input(path, True) + raise + finally: + if preview is not None: + bpy.data.images.remove(preview) + return cls(rgba, image.colorspace_settings.name, image.alpha_mode, image.source, path) + + def apply(self, target, alpha_path, *, prepare_undo=False): + target.validate() + image = target.image + if (not image.is_float or image.colorspace_settings.name != self.color_space + or image.alpha_mode != self.alpha_mode or image.source != self.source + or not np.array_equal(image_rgba(image), self.rgba)): + raise RuntimeError("The source HDR image changed; run Remove Background again") + try: + alpha = np.load(alpha_path, allow_pickle=False) + except (OSError, ValueError, EOFError) as error: + raise RuntimeError("Unable to read the background alpha result") from error + if not isinstance(alpha, np.ndarray): + alpha.close() + raise RuntimeError("Invalid background alpha result") + if (alpha.dtype != np.float32 or alpha.shape != self.rgba.shape[:2] + or not np.isfinite(alpha).all() or np.any((alpha < 0) | (alpha > 1))): + raise RuntimeError("Invalid background alpha result") + # Keep RGB independent of the new mask, including completely transparent pixels. + rgba = self.rgba.copy() + if self.alpha_mode == "NONE": + rgba[..., 3] = 1 + rgba[..., 3] *= alpha + with tempfile.TemporaryDirectory(prefix="anyimage-hdr-result-") as directory: + result = create_packed_hdr( + rgba, image.name, Path(directory) / "result.exr", alpha_mode="CHANNEL_PACKED", + ) + original_state = None + try: + if (prepare_undo and (image.is_dirty or image.source == "GENERATED") + and not has_other_image_user(target.owner)): + original_state = image_content_state(image) + baseline_rgba = self.rgba.copy() + if self.alpha_mode == "NONE": + baseline_rgba[..., 3] = 1 + baseline = create_packed_hdr( + baseline_rgba, image.name, Path(directory) / "source.exr", + alpha_mode="CHANNEL_PACKED" if self.alpha_mode == "CHANNEL_PACKED" else "PREMUL", + ) + try: + restore_image_content(image, image_content_state(baseline)) + finally: + bpy.data.images.remove(baseline) + if "FINISHED" not in bpy.ops.ed.undo_push(message="Before HDR Background Removal"): + raise RuntimeError("Unable to preserve the source HDR image for Undo") + except Exception: + bpy.data.images.remove(result) + if original_state is not None: + restore_image_content(image, original_state) + raise + try: + return target.commit(result, isolate_shared=True) + except Exception: + if original_state is not None: + restore_image_content(image, original_state) + raise diff --git a/src/anyimage/common/image.py b/src/anyimage/common/image.py index 697eec7..6b93146 100644 --- a/src/anyimage/common/image.py +++ b/src/anyimage/common/image.py @@ -8,6 +8,8 @@ from pathlib import Path import bpy +from ..server.media.input import validate_input_path + VIEW_TRANSFORM_COLOR_SPACES = ( ("ACES 1.3", "ACES 1.3 sRGB"), ("ACES 2.0", "ACES 2.0 sRGB"), @@ -333,11 +335,6 @@ def extract_frame_range(image, image_user): return start_frame, frame_count -def source_start_frame(source_object_name): - source_object = bpy.data.objects.get(source_object_name) - return extract_frame_range(source_object.data, source_object.image_user)[0] if source_object else 1 - - def image_path(image): filepath_from_user = getattr(image, "filepath_from_user", None) filepath = ( @@ -395,22 +392,19 @@ def normal_data_name(source_object): return f"{image_base_name(source_object.data)}_normal.png" -def color_data_name(source_object): - return f"{image_base_name(source_object.data)}_color.png" - - def save_packed_image(image, _scene): output_path = Path(tempfile.mkdtemp(prefix="anyimage-input-")) / "input.png" original_file_format = image.file_format try: image.file_format = "PNG" image.save(filepath=str(output_path), save_copy=True) - except (OSError, RuntimeError) as error: - output_path.unlink(missing_ok=True) + except (OSError, RuntimeError, ValueError) as error: + cleanup_image_input(output_path, True) raise RuntimeError("Unable to export the packed image") from error finally: image.file_format = original_file_format if not output_path.is_file(): + cleanup_image_input(output_path, True) raise RuntimeError("Blender did not export the packed image") return output_path @@ -450,9 +444,11 @@ def prepare_image_input(image, scene): if getattr(image, "source", None) == "MOVIE": if not source_path.is_file(): raise RuntimeError( - "The image empty must reference an existing video file" + f"Source video file does not exist: {source_path}" ) return source_path, False + if getattr(image, "source", None) == "SEQUENCE": + validate_input_path(source_path, images_only=True) sequence_frames = collect_sequence_frames(image) if sequence_frames: temporary_directory = Path(tempfile.mkdtemp(prefix="anyimage-input-")) @@ -469,73 +465,11 @@ def prepare_image_input(image, scene): return temporary_directory, True if not source_path.is_file(): raise RuntimeError( - "The image empty must reference an existing image file" + f"Source image file does not exist: {source_path}" ) return source_path, False -def prepare_image_bounds_input(image, source_rgba, bounds): - """Write one rectangular RGBA input owned by its temporary directory.""" - import numpy as np - - left, top, right, bottom = (int(value) for value in bounds) - rgba = np.asarray(source_rgba, dtype=np.float32) - expected_shape = (int(image.size[1]), int(image.size[0]), 4) - if rgba.shape != expected_shape: - raise ValueError("The source RGBA does not match the Blender Image") - if ( - left < 0 - or top < 0 - or right > expected_shape[1] - or bottom > expected_shape[0] - ): - raise ValueError("The image bounds are outside the Blender Image") - if right <= left or bottom <= top: - raise ValueError("The image bounds are empty") - cropped = rgba[top:bottom, left:right] - output_directory = Path(tempfile.mkdtemp(prefix="anyimage-bounds-")) - output_path = output_directory / "input.png" - temporary_image = bpy.data.images.new( - "AnyImage Bounds Input", - width=right - left, - height=bottom - top, - alpha=True, - ) - original_format = temporary_image.file_format - try: - temporary_image.colorspace_settings.name = image.colorspace_settings.name - temporary_image.alpha_mode = "STRAIGHT" - temporary_image.pixels.foreach_set(np.flipud(cropped).ravel()) - temporary_image.file_format = "PNG" - temporary_image.save(filepath=str(output_path), save_copy=True) - except Exception: - cleanup_image_input(output_path, True) - raise - finally: - temporary_image.file_format = original_format - bpy.data.images.remove(temporary_image, do_unlink=True) - if not output_path.is_file(): - cleanup_image_input(output_path, True) - raise RuntimeError("Blender did not export the bounds image input") - return output_path - - -def load_packed_rgba_image(path, source_image, name): - """Load an encoded local RGBA file and make its Blender data self-contained.""" - result_image = None - try: - result_image = bpy.data.images.load(str(path), check_existing=False) - result_image.colorspace_settings.name = source_image.colorspace_settings.name - result_image.alpha_mode = source_image.alpha_mode - result_image.name = name - result_image.pack() - except (OSError, RuntimeError, TypeError, ValueError) as error: - if result_image is not None: - bpy.data.images.remove(result_image, do_unlink=True) - raise RuntimeError("Unable to prepare the RGBA image result") from error - return result_image - - def configure_image_user( image_user, frame_count=None, @@ -674,6 +608,15 @@ def has_other_image_empty(source_object, *, removed_objects=()): ) +def require_conversion_source(operator): + """Validate the object and image captured when a conversion was submitted.""" + source = require_image_empty(operator.source_object_name) + if (str(source.as_pointer()) != operator.source_identity + or str(source.data.as_pointer()) != operator.image_identity): + raise ValueError("The conversion source image has changed") + return source + + def image_content_state(image): """Capture encoded content and unsaved pixels for an Image transaction.""" import numpy as np @@ -728,7 +671,7 @@ def restore_image_content(image, state): def replace_empty_image(source_object, result_image, *, placement_bounds=None, - removed_objects=()): + removed_objects=(), isolate_image=False): """Replace one Empty image, retaining shared source images.""" try: source_image = source_object.data @@ -767,7 +710,7 @@ def replace_empty_image(source_object, result_image, *, placement_bounds=None, ) original_content = None try: - shared = has_other_image_empty(source_object, removed_objects=removed_objects) + shared = isolate_image or has_other_image_empty(source_object, removed_objects=removed_objects) if shared: final_image = result_image else: @@ -866,22 +809,6 @@ def load_image_edit_result( ) -def load_color_result_image( - source_object, - output_directory, - *, - frame_prefix="image.", -): - """Load one packed Color texture result for a generated material.""" - return _load_packed_result_image( - source_object.data, - output_directory, - frame_prefix=frame_prefix, - result_name=color_data_name(source_object), - alpha_mode=color_alpha_mode(source_object.data.colorspace_settings.name), - ) - - def image_empty_bounds(source_object): image_width, image_height = source_object.data.size if image_width <= 0 or image_height <= 0: diff --git a/src/anyimage/common/image_target.py b/src/anyimage/common/image_target.py index b53f973..64b2034 100644 --- a/src/anyimage/common/image_target.py +++ b/src/anyimage/common/image_target.py @@ -1,6 +1,7 @@ """Resolve and commit AI image edits for Empty objects and texture nodes.""" from dataclasses import dataclass +from uuid import uuid4 import bpy @@ -56,16 +57,18 @@ def owner_image(owner): return owner.data if is_image_empty(owner) else owner.image -def has_other_texture_image_user(node): +def has_other_image_user(owner): """Check actual Image references, excluding editor display and Fake User.""" - tree = node.id_data - image = node.image + tree = None if is_image_empty(owner) else owner.id_data + image = owner_image(owner) for user in bpy.data.user_map(subset={image})[image]: if isinstance(user, bpy.types.Screen): continue - if user == tree or getattr(user, "node_tree", None) == tree: + if user == owner: + continue + if tree is not None and (user == tree or getattr(user, "node_tree", None) == tree): if any( - other != node and getattr(other, "image", None) == image + other != owner and getattr(other, "image", None) == image for other in tree.nodes ): return True @@ -83,7 +86,7 @@ def replace_texture_image(node, result_image): original_content = None frame_count = video_total_frames(result_image) if is_animated_image(result_image) else None try: - if has_other_texture_image_user(node): + if has_other_image_user(node): final_image = result_image else: result_content = image_content_state(result_image) @@ -121,6 +124,7 @@ class ImageEditTarget: tree: object = None start_frame: int = 1 frame_count: int = 1 + node_identity: str = "" @classmethod def capture(cls, context): @@ -129,10 +133,15 @@ class ImageEditTarget: raise RuntimeError("Select an Image Empty or a material Image Texture") image = owner_image(owner) start_frame, frame_count = extract_frame_range(image, owner.image_user) + tree = None if is_image_empty(owner) else owner.id_data + identity = "" + if tree is not None: + identity = owner.get("anyimage_identity") + if not identity: + identity = uuid4().hex + owner["anyimage_identity"] = identity return cls( - owner, image, - None if is_image_empty(owner) else owner.id_data, - start_frame, frame_count, + owner, image, tree, start_frame, frame_count, identity, ) def validate(self): @@ -146,6 +155,7 @@ class ImageEditTarget: and self.image.is_editable and self.tree.nodes.get(self.owner.name) == self.owner and self.owner.image == self.image + and self.owner.get("anyimage_identity") == self.node_identity ) if valid: return @@ -162,6 +172,19 @@ class ImageEditTarget: def apply(self, directory, *, frame_prefix="image."): self.validate() result_image = load_image_edit_result(self.image, directory, frame_prefix=frame_prefix) + return self.commit(result_image) + + def commit(self, result_image, *, isolate_shared=False): + """Commit an owned, fully prepared image through the target transaction.""" + try: + self.validate() + except Exception: + bpy.data.images.remove(result_image) + raise if self.tree is None: + if isolate_shared: + return replace_empty_image( + self.owner, result_image, isolate_image=has_other_image_user(self.owner), + ) return replace_empty_image(self.owner, result_image) return replace_texture_image(self.owner, result_image) diff --git a/src/anyimage/common/material.py b/src/anyimage/common/material.py index e03a0b2..f35fb56 100644 --- a/src/anyimage/common/material.py +++ b/src/anyimage/common/material.py @@ -6,10 +6,8 @@ from ..preferences import configured_material_view_adaptation from .image import ( VIEW_TRANSFORM_COLOR_SPACES, color_alpha_mode, - configure_image_user, image_base_name, image_pixels, - is_animated_image, ) from .node import load_node_group @@ -30,7 +28,9 @@ def material_color_space_name(scene): def configure_material_color_image(image, scene=None): - """Configure an owned material Color image using the current shading preference.""" + """Configure the selected Color image in place using the shading preference.""" + if getattr(image, "is_float", False): + return image if scene is None: scene = getattr(bpy.context, "scene", None) preferred = ( @@ -44,11 +44,9 @@ def configure_material_color_image(image, scene=None): or getattr(image, "is_dirty", False) else None ) - shared = image.users > int(image.use_fake_user) - result = image.copy() if shared else image - settings = result.colorspace_settings + settings = image.colorspace_settings original_space = settings.name - original_alpha = result.alpha_mode + original_alpha = image.alpha_mode try: for color_space in dict.fromkeys((preferred, "sRGB")): try: @@ -59,21 +57,18 @@ def configure_material_color_image(image, scene=None): break else: raise RuntimeError("Unable to configure the material color space") - result.alpha_mode = color_alpha_mode(settings.name) + image.alpha_mode = color_alpha_mode(settings.name) if pixels is not None: - result.pixels.foreach_set(pixels) - result.update() + image.pixels.foreach_set(pixels) + image.update() except Exception: - if shared: - bpy.data.images.remove(result) - else: - settings.name = original_space - result.alpha_mode = original_alpha - if pixels is not None: - result.pixels.foreach_set(pixels) - result.update() + settings.name = original_space + image.alpha_mode = original_alpha + if pixels is not None: + image.pixels.foreach_set(pixels) + image.update() raise - return result + return image def material_node_group(depth_plane=False): @@ -93,178 +88,167 @@ def create_image_material( displacement_midlevel=0.0, displacement_scale=1.0, shadeless=False, - frame_count=1, - frame_start=1, - color_frame_offset=0, - color_frame_duration=None, - depth_frame_offset=0, - depth_frame_duration=None, - use_cyclic=None, normal_image=None, normal_space="TANGENT", scene=None, + texture_extension="EXTEND", ): - material_image = configure_material_color_image(color_image, scene) + preserve_color = color_image.is_float or ( + shadeless and not color_image.colorspace_settings.name.endswith("sRGB") + ) + material_image = color_image if preserve_color else configure_material_color_image(color_image, scene) + if material_image.source == "GENERATED" or material_image.is_dirty: + material_image.pack() material = bpy.data.materials.new(image_base_name(source_image)) - material.use_nodes = True - if hasattr(material, "surface_render_method"): - material.surface_render_method = "DITHERED" - nodes = material.node_tree.nodes - nodes.clear() + try: + material.use_nodes = True + if hasattr(material, "surface_render_method"): + material.surface_render_method = "DITHERED" + nodes = material.node_tree.nodes + nodes.clear() - color_texture = nodes.new("ShaderNodeTexImage") - uv_map = nodes.new("ShaderNodeUVMap") - output = nodes.new("ShaderNodeOutputMaterial") + color_texture = nodes.new("ShaderNodeTexImage") + uv_map = nodes.new("ShaderNodeUVMap") + output = nodes.new("ShaderNodeOutputMaterial") - uv_map.uv_map = "UVMap" - uv_map.location = (-1250.0, 350.0) - color_texture.image = material_image - color_texture.interpolation = "Linear" - color_texture.extension = "EXTEND" - color_texture.location = (-1000.0, 350.0) - output.location = (850.0, 350.0) - extracted_frames = frame_count if frame_count > 1 else None - color_frames = ( - color_frame_duration if color_frame_duration is not None else extracted_frames - ) - depth_frames = ( - depth_frame_duration if depth_frame_duration is not None else extracted_frames - ) - material_frames = color_frames - material_frame_offset = color_frame_offset - if frame_count > 1 or is_animated_image(material_image): - configure_image_user( - color_texture.image_user, - material_frames, - frame_start=frame_start, - frame_offset=material_frame_offset, - use_cyclic=use_cyclic, - ) - - links = material.node_tree.links - links.new(uv_map.outputs["UV"], color_texture.inputs["Vector"]) - alpha = color_texture.outputs["Alpha"] - material_inputs = nodes.new("ShaderNodeGroup") - is_depth_plane = displacement_image is not None - material_inputs.node_tree = material_node_group(is_depth_plane) - material_inputs.location = (-300.0, 350.0) - material_inputs.inputs["Bump Scale"].default_value = depth_scale - if not is_depth_plane: - material_inputs.inputs["Alpha Fix"].default_value = float( - configured_material_view_adaptation() - ) - if "Object Space" in material_inputs.inputs: - if normal_space not in {"TANGENT", "OBJECT"}: - raise ValueError(f"Unsupported normal space: {normal_space}") - material_inputs.inputs["Object Space"].default_value = normal_space == "OBJECT" - links.new(color_texture.outputs["Color"], material_inputs.inputs["Color"]) - links.new(alpha, material_inputs.inputs["Alpha"]) - if normal_image is not None and "Normal" in material_inputs.inputs: - normal_texture = nodes.new("ShaderNodeTexImage") - normal_texture.image = normal_image - normal_texture.interpolation = "Linear" - normal_texture.extension = "EXTEND" - normal_texture.location = (-1000.0, -50.0) - links.new(uv_map.outputs["UV"], normal_texture.inputs["Vector"]) - links.new(normal_texture.outputs["Color"], material_inputs.inputs["Normal"]) - if shadeless: - shader = nodes.new("ShaderNodeGroup") - shader.node_tree = load_node_group(SHADELESS_NODE_GROUP_NAME) - shader.name = SHADELESS_NODE_GROUP_NAME - transparent = nodes.new("ShaderNodeBsdfTransparent") - mix_shader = nodes.new("ShaderNodeMixShader") - shader.location = (0.0, 550.0) - transparent.location = (0.0, 150.0) - mix_shader.location = (300.0, 350.0) - links.new( - material_inputs.outputs["Color"], - shader.inputs["Color"], - ) - links.new( - material_inputs.outputs["Alpha"], - mix_shader.inputs["Fac"], - ) - links.new( - transparent.outputs["BSDF"], - mix_shader.inputs[1], - ) - links.new( - shader.outputs["Shader"], - mix_shader.inputs[2], - ) - links.new( - mix_shader.outputs["Shader"], - output.inputs["Surface"], - ) - else: - shader = nodes.new("ShaderNodeBsdfPrincipled") - shader.location = (250.0, 400.0) - shader.inputs["IOR"].default_value = 1.2 - - links.new( - material_inputs.outputs["Color"], - shader.inputs["Base Color"], - ) - links.new( - material_inputs.outputs["Color"], - shader.inputs["Subsurface Radius"], - ) - links.new( - material_inputs.outputs["Color"], - shader.inputs["Emission Color"], - ) - links.new(material_inputs.outputs["Alpha"], shader.inputs["Alpha"]) - links.new(material_inputs.outputs["Normal"], shader.inputs["Normal"]) - links.new( - material_inputs.outputs["Normal"], - shader.inputs["Coat Normal"], - ) - links.new( - material_inputs.outputs["Roughness"], - shader.inputs["Roughness"], - ) - links.new( - material_inputs.outputs["Color"], - shader.inputs["Specular IOR Level"], - ) - links.new(shader.outputs["BSDF"], output.inputs["Surface"]) - - if displacement_image is not None: - depth_texture = nodes.new("ShaderNodeTexImage") - depth_texture.image = displacement_image - depth_texture.interpolation = "Linear" - depth_texture.extension = "EXTEND" - if frame_count > 1 or is_animated_image(displacement_image): - configure_image_user( - depth_texture.image_user, - depth_frames, - frame_start=frame_start, - frame_offset=depth_frame_offset, - use_cyclic=use_cyclic, + uv_map.uv_map = "UVMap" + uv_map.location = (-1250.0, 350.0) + color_texture.image = material_image + color_texture.interpolation = "Linear" + color_texture.extension = texture_extension + color_texture.location = (-1000.0, 350.0) + output.location = (850.0, 350.0) + links = material.node_tree.links + links.new(uv_map.outputs["UV"], color_texture.inputs["Vector"]) + if texture_extension == "REPEAT": + axes = nodes.new("ShaderNodeSeparateXYZ") + axes.location = (-1650.0, 350.0) + clamp = nodes.new("ShaderNodeClamp") + clamp.location = (-1450.0, 100.0) + half_pixel = 0.5 / max(1, int(material_image.size[1])) + clamp.inputs["Min"].default_value = half_pixel + clamp.inputs["Max"].default_value = 1.0 - half_pixel + coordinates = nodes.new("ShaderNodeCombineXYZ") + coordinates.location = (-1200.0, 350.0) + uv_map.location = (-1900.0, 350.0) + links.new(uv_map.outputs["UV"], axes.inputs[0]) + links.new(axes.outputs["Y"], clamp.inputs["Value"]) + links.new(axes.outputs["X"], coordinates.inputs["X"]) + links.new(clamp.outputs[0], coordinates.inputs["Y"]) + links.new(coordinates.outputs[0], color_texture.inputs["Vector"]) + alpha = color_texture.outputs["Alpha"] + if shadeless and displacement_image is None: + shader = nodes.new("ShaderNodeGroup") + shader.node_tree = load_node_group(SHADELESS_NODE_GROUP_NAME) + shader.location = (0.0, 350.0) + links.new(color_texture.outputs["Color"], shader.inputs["Color"]) + links.new(alpha, shader.inputs["Alpha"]) + links.new(shader.outputs["Shader"], output.inputs["Surface"]) + return material + material_inputs = nodes.new("ShaderNodeGroup") + is_depth_plane = displacement_image is not None + material_inputs.node_tree = material_node_group(is_depth_plane) + material_inputs.location = (-300.0, 350.0) + material_inputs.inputs["Bump Scale"].default_value = depth_scale + if not is_depth_plane: + material_inputs.inputs["Alpha Fix"].default_value = float( + not material_image.is_float and configured_material_view_adaptation() + ) + if "Object Space" in material_inputs.inputs: + if normal_space not in {"TANGENT", "OBJECT"}: + raise ValueError(f"Unsupported normal space: {normal_space}") + material_inputs.inputs["Object Space"].default_value = normal_space == "OBJECT" + links.new(color_texture.outputs["Color"], material_inputs.inputs["Color"]) + links.new(alpha, material_inputs.inputs["Alpha"]) + if normal_image is not None and "Normal" in material_inputs.inputs: + normal_texture = nodes.new("ShaderNodeTexImage") + normal_texture.image = normal_image + normal_texture.interpolation = "Linear" + normal_texture.extension = "EXTEND" + normal_texture.location = (-1000.0, -50.0) + links.new(uv_map.outputs["UV"], normal_texture.inputs["Vector"]) + links.new(normal_texture.outputs["Color"], material_inputs.inputs["Normal"]) + if shadeless: + shader = nodes.new("ShaderNodeGroup") + shader.node_tree = load_node_group(SHADELESS_NODE_GROUP_NAME) + shader.name = SHADELESS_NODE_GROUP_NAME + shader.location = (0.0, 550.0) + links.new( + material_inputs.outputs["Color"], + shader.inputs["Color"], + ) + links.new( + material_inputs.outputs["Alpha"], + shader.inputs["Alpha"], + ) + links.new( + shader.outputs["Shader"], + output.inputs["Surface"], ) - depth_texture.location = (-1000.0, -400.0) - links.new(uv_map.outputs["UV"], depth_texture.inputs["Vector"]) - displacement_height = depth_texture.outputs["Color"] - if displacement_midlevel != 0.0 or displacement_scale != 1.0: - depth_adjust = nodes.new("ShaderNodeMath") - depth_adjust.operation = "SUBTRACT" - depth_adjust.inputs[1].default_value = displacement_midlevel - depth_adjust.location = (500.0, -550.0) - depth_scale_node = nodes.new("ShaderNodeMath") - depth_scale_node.operation = "MULTIPLY" - depth_scale_node.inputs[1].default_value = displacement_scale - depth_scale_node.location = (700.0, -550.0) - links.new(displacement_height, depth_adjust.inputs[0]) - links.new(depth_adjust.outputs[0], depth_scale_node.inputs[0]) - links.new(depth_scale_node.outputs[0], material_inputs.inputs["Depth"]) else: - links.new(displacement_height, material_inputs.inputs["Depth"]) - links.new( - material_inputs.outputs["Displacement"], - output.inputs["Displacement"], - ) - set_displacement_only(material) - return material + shader = nodes.new("ShaderNodeBsdfPrincipled") + shader.location = (250.0, 400.0) + shader.inputs["IOR"].default_value = 1.2 + + links.new( + material_inputs.outputs["Color"], + shader.inputs["Base Color"], + ) + links.new( + material_inputs.outputs["Color"], + shader.inputs["Subsurface Radius"], + ) + links.new( + material_inputs.outputs["Color"], + shader.inputs["Emission Color"], + ) + links.new(material_inputs.outputs["Alpha"], shader.inputs["Alpha"]) + links.new(material_inputs.outputs["Normal"], shader.inputs["Normal"]) + links.new( + material_inputs.outputs["Normal"], + shader.inputs["Coat Normal"], + ) + links.new( + material_inputs.outputs["Roughness"], + shader.inputs["Roughness"], + ) + links.new( + material_inputs.outputs["Color"], + shader.inputs["Specular IOR Level"], + ) + links.new(shader.outputs["BSDF"], output.inputs["Surface"]) + + if displacement_image is not None: + depth_texture = nodes.new("ShaderNodeTexImage") + depth_texture.image = displacement_image + depth_texture.interpolation = "Linear" + depth_texture.extension = "EXTEND" + depth_texture.location = (-1000.0, -400.0) + links.new(uv_map.outputs["UV"], depth_texture.inputs["Vector"]) + displacement_height = depth_texture.outputs["Color"] + if displacement_midlevel != 0.0 or displacement_scale != 1.0: + depth_adjust = nodes.new("ShaderNodeMath") + depth_adjust.operation = "SUBTRACT" + depth_adjust.inputs[1].default_value = displacement_midlevel + depth_adjust.location = (500.0, -550.0) + depth_scale_node = nodes.new("ShaderNodeMath") + depth_scale_node.operation = "MULTIPLY" + depth_scale_node.inputs[1].default_value = displacement_scale + depth_scale_node.location = (700.0, -550.0) + links.new(displacement_height, depth_adjust.inputs[0]) + links.new(depth_adjust.outputs[0], depth_scale_node.inputs[0]) + links.new(depth_scale_node.outputs[0], material_inputs.inputs["Depth"]) + else: + links.new(displacement_height, material_inputs.inputs["Depth"]) + links.new( + material_inputs.outputs["Displacement"], + output.inputs["Displacement"], + ) + set_displacement_only(material) + return material + except Exception: + bpy.data.materials.remove(material, do_unlink=True) + raise def set_displacement_only(material): diff --git a/src/anyimage/menu.py b/src/anyimage/menu.py index b823c38..f8c0565 100644 --- a/src/anyimage/menu.py +++ b/src/anyimage/menu.py @@ -27,6 +27,11 @@ class AnyImageImageMenu(bpy.types.Menu): text="Convert to Relief Plane", icon="MOD_DISPLACE", ) + layout.operator( + "anyimage.convert_to_panorama", + text="Convert to Panorama", + icon="SPHERE", + ) layout.separator() draw_image_actions(layout, context) diff --git a/src/anyimage/operators/__init__.py b/src/anyimage/operators/__init__.py index 99dc313..fe90d6f 100644 --- a/src/anyimage/operators/__init__.py +++ b/src/anyimage/operators/__init__.py @@ -20,9 +20,11 @@ from .ai_setup import ( SetupAIEnvironment, ) from .upscale import UpscaleImage +from .convert_to_panorama import CLASSES as PANORAMA_CLASSES CLASSES = ( + *PANORAMA_CLASSES, SetupAIEnvironment, OpenAIEnvironmentSettings, DownloadModel, diff --git a/src/anyimage/operators/ai_setup.py b/src/anyimage/operators/ai_setup.py index 800e85a..a311a8d 100644 --- a/src/anyimage/operators/ai_setup.py +++ b/src/anyimage/operators/ai_setup.py @@ -8,7 +8,7 @@ from ..properties import ( model_download_size, model_record, ) -from ..common.ai import require_environment +from ..common.ai import report_ai_error, require_environment def format_download_size(size): @@ -94,14 +94,18 @@ class SetupAIEnvironment(bpy.types.Operator): status = ai_status() if status["ready"]: return {"FINISHED"} - if status["environment_ready"]: - return bpy.ops.anyimage.download_required_models("INVOKE_DEFAULT") - installer = ( - bpy.ops.anyimage.install_environment_mirror - if self.use_mirror - else bpy.ops.anyimage.install_environment - ) - return installer("INVOKE_DEFAULT") + try: + if status["environment_ready"]: + return bpy.ops.anyimage.download_required_models("INVOKE_DEFAULT") + installer = ( + bpy.ops.anyimage.install_environment_mirror + if self.use_mirror + else bpy.ops.anyimage.install_environment + ) + return installer("INVOKE_DEFAULT") + except RuntimeError as error: + report_ai_error(self, error) + return {"CANCELLED"} class OpenAIEnvironmentSettings(bpy.types.Operator): diff --git a/src/anyimage/operators/clipboard_image/actions.py b/src/anyimage/operators/clipboard_image/actions.py index 7526f38..0ddd3c0 100644 --- a/src/anyimage/operators/clipboard_image/actions.py +++ b/src/anyimage/operators/clipboard_image/actions.py @@ -5,8 +5,11 @@ import tempfile import bpy from ...common.material import create_image_material +from ...common.color_image import material_color_image +from ...common.image import image_pixels CLIPBOARD_HASH_PROPERTY = "anyimage_clipboard_sha256" +CLIPBOARD_CONTENT_PROPERTY = "anyimage_clipboard_content_sha256" CLIPBOARD_TEXTURE_PROPERTY = "anyimage_clipboard_texture" DEFAULT_STENCIL_SIZE = 256.0 REFERENCE_IMAGE_SIZE = 5.0 @@ -80,8 +83,9 @@ def acquire_packed_image(image_data, suffix): image.name = "Clipboard" image.pack() image[CLIPBOARD_HASH_PROPERTY] = image_hash + image[CLIPBOARD_CONTENT_PROPERTY] = _clipboard_content_hash(image) return image, False - except (OSError, RuntimeError): + except Exception: if image is not None: bpy.data.images.remove(image) raise @@ -96,11 +100,27 @@ def _find_packed_clipboard_image(image_hash): for image in bpy.data.images: if image.get(CLIPBOARD_HASH_PROPERTY) != image_hash: continue - if _is_image_packed(image): - return image + if not _is_image_packed(image): + continue + baseline = image.get(CLIPBOARD_CONTENT_PROPERTY) + if not baseline: + continue + try: + if _clipboard_content_hash(image) == baseline: + return image + except (RuntimeError, ValueError, ReferenceError, OSError): + continue return None +def _clipboard_content_hash(image): + """Fingerprint current pixels and interpretation without changing the image.""" + digest = hashlib.sha256() + digest.update(repr((tuple(image.size), image.colorspace_settings.name, image.alpha_mode)).encode()) + digest.update(image_pixels(image).tobytes()) + return digest.hexdigest() + + def _is_image_packed(image): if getattr(image, "packed_file", None) is not None: return True @@ -150,17 +170,24 @@ def add_image_plane( from ..convert_to_plane.image_plane import create_image_plane_object half_width, half_height = _plane_half_dimensions(width, height) - material = create_image_material( - image, image, shadeless=shadeless, scene=context.scene - ) - plane = create_image_plane_object( - context, - image.name, - (-half_width, half_width, -half_height, half_height), - material, - ) - _place_in_view(context, plane, location) - _select_only(context, plane) + material = plane = None + try: + with material_color_image(image) as color: + material = create_image_material(image, color, shadeless=shadeless, scene=context.scene) + plane = create_image_plane_object( + context, image.name, (-half_width, half_width, -half_height, half_height), material, + ) + _place_in_view(context, plane, location) + _select_only(context, plane) + except Exception: + if plane is not None: + mesh = plane.data + bpy.data.objects.remove(plane, do_unlink=True) + if mesh.users == 0: + bpy.data.meshes.remove(mesh) + if material is not None and material.users == 0: + bpy.data.materials.remove(material) + raise return plane diff --git a/src/anyimage/operators/convert_to_panorama/__init__.py b/src/anyimage/operators/convert_to_panorama/__init__.py new file mode 100644 index 0000000..30aba41 --- /dev/null +++ b/src/anyimage/operators/convert_to_panorama/__init__.py @@ -0,0 +1,6 @@ +"""Image Empty panorama conversion operators.""" + +from .operators import ConvertToPanorama, GeneratePanorama + +CLASSES = (GeneratePanorama, ConvertToPanorama) +__all__ = ("ConvertToPanorama", "GeneratePanorama", "CLASSES") diff --git a/src/anyimage/operators/convert_to_panorama/object.py b/src/anyimage/operators/convert_to_panorama/object.py new file mode 100644 index 0000000..3cf51e5 --- /dev/null +++ b/src/anyimage/operators/convert_to_panorama/object.py @@ -0,0 +1,71 @@ +"""Apply packed panorama artifacts around the source Empty's camera origin.""" + +import json +import math +from pathlib import Path + +import bpy + +from ...common.depth import load_depth_result_image +from ...common.image import require_conversion_source +from ...common.color_image import material_color_image +from ...common.material import create_image_material +from ...common.node import load_node_group +from ...common.object import finalize_object_result, modifier_input_identifier, set_modifier_input + + +def load_panorama_metadata(path): + metadata = json.loads(Path(path).read_text(encoding="utf-8")) + try: + size = tuple(int(value) for value in metadata["image_size"]) + distance = float(metadata["reference_distance"]) + valid = (metadata["projection"] == "equirectangular" and len(size) == 2 + and size[1] > 0 and size[0] == 2 * size[1] + and math.isfinite(distance) and distance > 0) + except (KeyError, TypeError, ValueError) as error: + raise ValueError("Panorama metadata is invalid") from error + if not valid: + raise ValueError("Panorama metadata is invalid") + return {"image_size": size, "reference_distance": distance} + + +def create_panorama_from_result(context, result, operator): + source = require_conversion_source(operator) + depth = material = mesh = obj = None + try: + with material_color_image(input_path=operator.color_path) as color: + metadata = load_panorama_metadata(result.file("depth_metadata")) + group = load_node_group("O Image Depth Panorama") + depth = load_depth_result_image(result.file("depth"), source, metadata) + material = create_image_material(source.data, color, shadeless=True, scene=context.scene, + texture_extension="REPEAT") + mesh = bpy.data.meshes.new(source.name) + mesh.from_pydata([(0, 0, 0)], [], []) + mesh.materials.append(material) + obj = bpy.data.objects.new(source.name, mesh) + context.collection.objects.link(obj) + obj.location = source.matrix_world.translation + obj.scale = source.matrix_world.to_scale() + obj["anyimage_mesh_shape"] = "PANORAMA" + modifier = obj.modifiers.new(group.name, "NODES") + modifier.node_group = group + modifier.show_group_selector = False + for name, value in ( + ("Subdivide", operator.mesh_detail), ("Depth Image", depth), + ("Reference Distance", metadata["reference_distance"]), + ): + set_modifier_input(modifier, modifier_input_identifier(group, name), value) + finalize_object_result(context, source, obj) + except Exception: + if obj is not None: + bpy.data.objects.remove(obj, do_unlink=True) + if mesh is not None and mesh.users == 0: + bpy.data.meshes.remove(mesh) + if material is not None and material.users == 0: + bpy.data.materials.remove(material, do_unlink=True) + if depth is not None and depth.users == 0: + bpy.data.images.remove(depth, do_unlink=True) + raise + if "FINISHED" not in bpy.ops.ed.undo_push(message="Convert to Panorama"): + raise RuntimeError("Unable to create the Convert to Panorama Undo step") + return obj diff --git a/src/anyimage/operators/convert_to_panorama/operators.py b/src/anyimage/operators/convert_to_panorama/operators.py new file mode 100644 index 0000000..25576af --- /dev/null +++ b/src/anyimage/operators/convert_to_panorama/operators.py @@ -0,0 +1,96 @@ +"""Submit the current static panorama through the managed geometry Job.""" + +import bpy + +from ...common.ai import invoke_ai_setup_if_needed, production_device, report_ai_error, require_environment, require_input_path, require_model +from ...common.image import is_image_empty, video_total_frames +from ...common.color_image import prepare_material_color_input, material_analysis_input, cleanup_material_color_input +from ...preferences import addon_preferences, configured_max_ai_input_size +from ...properties import DEFAULT_MOGE2_MODEL_KEY +from ...runtime import JobOperatorBase +from ...server.media.resolution import DEFAULT_MAX_AI_INPUT_SIZE, MAX_MAX_AI_INPUT_SIZE, MIN_MAX_AI_INPUT_SIZE +from .object import create_panorama_from_result + +_HIDDEN = {"HIDDEN", "SKIP_SAVE"} + + +class GeneratePanorama(JobOperatorBase, bpy.types.Operator): + bl_idname = "anyimage.generate_panorama" + bl_label = "Generate Panorama" + job_type = "generate-panorama-geometry" + starting_message = "Preparing panorama conversion" + + input_path: bpy.props.StringProperty(options=_HIDDEN) + color_path: bpy.props.StringProperty(options=_HIDDEN) + source_object_name: bpy.props.StringProperty(options=_HIDDEN) + source_identity: bpy.props.StringProperty(options=_HIDDEN) + image_identity: bpy.props.StringProperty(options=_HIDDEN) + mesh_detail: bpy.props.IntProperty(default=7, min=0, max=8, options=_HIDDEN) + model: bpy.props.StringProperty(default=DEFAULT_MOGE2_MODEL_KEY, options=_HIDDEN) + resolution_level: bpy.props.IntProperty(default=5, min=0, max=9, options=_HIDDEN) + max_input_size: bpy.props.IntProperty(default=DEFAULT_MAX_AI_INPUT_SIZE, + min=MIN_MAX_AI_INPUT_SIZE, max=MAX_MAX_AI_INPUT_SIZE, options=_HIDDEN) + + def request(self, context): + require_environment() + require_model(self.model) + return { + "input": str(require_input_path(self.input_path)), + "device": production_device().lower(), "model": self.model, + "resolution_level": int(self.resolution_level), + "max_input_size": int(self.max_input_size), "video_frames": 1, "start_frame": 1, + } + + def response(self, context, result): + create_panorama_from_result(context, result, self) + return "Panorama is ready" + + def cleanup(self): + cleanup_material_color_input(self.color_path) + + +class ConvertToPanorama(bpy.types.Operator): + bl_idname = "anyimage.convert_to_panorama" + bl_label = "Convert to Panorama" + bl_description = "Convert a static 2:1 equirectangular Image Empty into a radial depth surface" + bl_options = {"REGISTER", "UNDO"} + + mesh_detail: bpy.props.IntProperty(name="Subdivide", default=7, min=0, max=8) + + @classmethod + def poll(cls, context): + from ...runtime import runtime + + return is_image_empty(context.object) and not runtime.server_busy() + + def execute(self, context): + source = context.object + image = source.data + width, height = image.size + if image.source in {"MOVIE", "SEQUENCE"} or video_total_frames(image) != 1 or height <= 0 or width != height * 2: + self.report({"ERROR"}, "Panorama conversion requires a static 2:1 equirectangular image") + return {"CANCELLED"} + setup = invoke_ai_setup_if_needed(self) + if setup is not None: + return setup + color_path = None + try: + preferences = addon_preferences(context) + color_path = prepare_material_color_input(image) + path = material_analysis_input(color_path) + result = bpy.ops.anyimage.generate_panorama( + input_path=str(path), color_path=str(color_path), source_object_name=source.name, + source_identity=str(source.as_pointer()), + image_identity=str(image.as_pointer()), mesh_detail=self.mesh_detail, + model=getattr(preferences, "geometry_model", DEFAULT_MOGE2_MODEL_KEY), + resolution_level=int(getattr(preferences, "geometry_resolution_level", 5)), + max_input_size=configured_max_ai_input_size(context), + ) + except (OSError, RuntimeError, ValueError) as error: + cleanup_material_color_input(color_path) + report_ai_error(self, error) + return {"CANCELLED"} + if "CANCELLED" in result: + cleanup_material_color_input(color_path) + return {"CANCELLED"} + return {"FINISHED"} diff --git a/src/anyimage/operators/convert_to_plane/image_plane.py b/src/anyimage/operators/convert_to_plane/image_plane.py index 1943347..86c473f 100644 --- a/src/anyimage/operators/convert_to_plane/image_plane.py +++ b/src/anyimage/operators/convert_to_plane/image_plane.py @@ -78,17 +78,25 @@ def create_image_plane_object( thickness=0.0, ): mesh = create_image_plane_mesh(name, bounds) - mesh.materials.append(material) - plane = bpy.data.objects.new(name, mesh) - context.collection.objects.link(plane) - if matrix_world is None: - from mathutils import Matrix + plane = None + try: + mesh.materials.append(material) + plane = bpy.data.objects.new(name, mesh) + context.collection.objects.link(plane) + if matrix_world is None: + from mathutils import Matrix - matrix_world = Matrix.Identity(4) - plane.matrix_world = build_image_plane_matrix(matrix_world, bounds) - add_image_plane_processing( - plane, - subdivisions=subdivisions, - thickness=thickness, - ) + matrix_world = Matrix.Identity(4) + plane.matrix_world = build_image_plane_matrix(matrix_world, bounds) + add_image_plane_processing( + plane, + subdivisions=subdivisions, + thickness=thickness, + ) + except Exception: + if plane is not None: + bpy.data.objects.remove(plane, do_unlink=True) + if mesh.users == 0: + bpy.data.meshes.remove(mesh) + raise return plane diff --git a/src/anyimage/operators/convert_to_plane/object.py b/src/anyimage/operators/convert_to_plane/object.py index d5c5b48..c2bb67d 100644 --- a/src/anyimage/operators/convert_to_plane/object.py +++ b/src/anyimage/operators/convert_to_plane/object.py @@ -3,13 +3,9 @@ import bpy from ...common.image import ( - color_data_name, - configure_color_alpha, image_empty_bounds, - image_pixels, - load_color_result_image, load_normal_result_image, - require_image_empty, + require_conversion_source, ) from ...common.depth import ( depth_uniform_scale, @@ -17,6 +13,7 @@ from ...common.depth import ( load_depth_result_image, ) from ...common.material import create_image_material +from ...common.color_image import material_color_image from ...common.object import ( finalize_object_result, modifier_input_identifier, @@ -30,20 +27,6 @@ from .image_plane import ( ) -def create_color_image(source_object): - source = source_object.data - image = source.copy() - if getattr(source, "source", None) == "GENERATED" or getattr( - source, "is_dirty", False - ): - pixels = image_pixels(source) - image.pixels.foreach_set(pixels) - image.update() - image.name = color_data_name(source_object) - configure_color_alpha(image) - return image - - def create_plane_object(context, source_object, subdivisions, material): source_name = source_object.name plane = create_image_plane_object( @@ -54,10 +37,17 @@ def create_plane_object(context, source_object, subdivisions, material): matrix_world=source_object.matrix_world, subdivisions=subdivisions, ) - for polygon in plane.data.polygons: - polygon.use_smooth = True - plane["anyimage_mesh_shape"] = "PLANE" - return finalize_object_result(context, source_object, plane) + try: + for polygon in plane.data.polygons: + polygon.use_smooth = True + plane["anyimage_mesh_shape"] = "PLANE" + return finalize_object_result(context, source_object, plane) + except Exception: + mesh = plane.data + bpy.data.objects.remove(plane, do_unlink=True) + if mesh.users == 0: + bpy.data.meshes.remove(mesh) + raise def create_depth_plane_object( @@ -102,54 +92,43 @@ def create_depth_plane_object( modifier_input_identifier(node_group, name), value, ) + return finalize_object_result(context, source_object, plane) except Exception: if plane is not None: bpy.data.objects.remove(plane, do_unlink=True) if mesh.users == 0: bpy.data.meshes.remove(mesh) raise - return finalize_object_result(context, source_object, plane) def create_depth_plane_from_result(context, result, operator): normal_space = {"DEPTH": "OBJECT", "RELIEF": "TANGENT"}[operator.plane_type] - source_object = require_image_empty(operator.source_object_name) - color_image = load_color_result_image(source_object, result.directory) - color_image.name = color_data_name(source_object) + source_object = require_conversion_source(operator) material = None depth_image = None normal_image = None try: - depth_metadata = load_depth_metadata(result.file("depth_metadata")) - depth_image = load_depth_result_image( - result.file("depth"), - source_object, - depth_metadata, - ) - normal_image = load_normal_result_image( - result.file(f"{normal_space.lower()}_normal"), - source_object, - ) - material = create_image_material( - source_object.data, - color_image, - normal_image=normal_image, - normal_space=normal_space, - scene=context.scene, - ) - create_depth_plane_object( - context, - source_object, - operator.mesh_detail, - material, - depth_image, - depth_metadata, - operator.plane_type, - ) + with material_color_image(input_path=operator.color_path) as color_image: + depth_metadata = load_depth_metadata(result.file("depth_metadata")) + depth_image = load_depth_result_image( + result.file("depth"), source_object, depth_metadata, + ) + normal_image = load_normal_result_image( + result.file(f"{normal_space.lower()}_normal"), source_object, + ) + material = create_image_material( + source_object.data, color_image, + normal_image=normal_image, normal_space=normal_space, + scene=context.scene, + ) + create_depth_plane_object( + context, source_object, operator.mesh_detail, material, + depth_image, depth_metadata, operator.plane_type, + ) except Exception: if material is not None and material.users == 0: bpy.data.materials.remove(material, do_unlink=True) - for image in (color_image, depth_image, normal_image): + for image in (depth_image, normal_image): if image is not None and image.users == 0: bpy.data.images.remove(image, do_unlink=True) raise diff --git a/src/anyimage/operators/convert_to_plane/operators.py b/src/anyimage/operators/convert_to_plane/operators.py index ba78c0a..c979610 100644 --- a/src/anyimage/operators/convert_to_plane/operators.py +++ b/src/anyimage/operators/convert_to_plane/operators.py @@ -15,18 +15,18 @@ from ...common.ai import ( require_environment, require_input_path, require_model, + report_ai_error, ) -from ...common.image import ( - cleanup_image_input, - image_user_settings, - is_image_empty, - prepare_image_input, - source_start_frame, - video_total_frames, -) +from ...common.image import is_image_empty from ...common.material import create_image_material +from ...common.color_image import ( + cleanup_material_color_input, + material_analysis_input, + material_color_image, + prepare_material_color_input, + require_static_color_image, +) from .object import ( - create_color_image, create_depth_plane_from_result, create_plane_object, ) @@ -49,7 +49,10 @@ class GenerateDepthPlane(JobOperatorBase, bpy.types.Operator): starting_message = "Preparing depth conversion" input_path: bpy.props.StringProperty(options=_HIDDEN) + color_path: bpy.props.StringProperty(options=_HIDDEN) source_object_name: bpy.props.StringProperty(options=_HIDDEN) + source_identity: bpy.props.StringProperty(options=_HIDDEN) + image_identity: bpy.props.StringProperty(options=_HIDDEN) plane_type: bpy.props.EnumProperty( items=(("DEPTH", "Depth Plane", ""), ("RELIEF", "Relief Plane", "")), default="DEPTH", @@ -64,7 +67,6 @@ class GenerateDepthPlane(JobOperatorBase, bpy.types.Operator): max=MAX_MAX_AI_INPUT_SIZE, options=_HIDDEN, ) - delete_input: bpy.props.BoolProperty(default=False, options=_HIDDEN) def request(self, context): require_environment() @@ -74,7 +76,7 @@ class GenerateDepthPlane(JobOperatorBase, bpy.types.Operator): "input": str(input_path), "device": production_device().lower(), "video_frames": 1, - "start_frame": source_start_frame(self.source_object_name), + "start_frame": 1, "max_input_size": int(self.max_input_size), "plane_type": self.plane_type, } @@ -84,7 +86,7 @@ class GenerateDepthPlane(JobOperatorBase, bpy.types.Operator): input_path, self.model, self.resolution_level, - source_start_frame(self.source_object_name), + 1, ) ) return parameters @@ -94,38 +96,42 @@ class GenerateDepthPlane(JobOperatorBase, bpy.types.Operator): return f"{self.plane_type.title()} Plane is ready" def cleanup(self): - cleanup_image_input(self.input_path, self.delete_input) + cleanup_material_color_input(self.color_path) def start_depth_plane_job(operator, context, plane_type): source_object = context.object - if video_total_frames(source_object.data) != 1: - operator.report({"ERROR"}, "Depth conversion currently supports still images") - return {"CANCELLED"} - setup_result = invoke_ai_setup_if_needed() - if setup_result is not None: - return setup_result try: - source_path, delete_input = prepare_image_input( - source_object.data, - context.scene, - ) - model, resolution_level = depth_plane_settings(context) - except (RuntimeError, ValueError) as error: + require_static_color_image(source_object.data) + except ValueError as error: operator.report({"ERROR"}, str(error)) return {"CANCELLED"} - result = bpy.ops.anyimage.generate_depth_plane( - input_path=str(source_path), - source_object_name=source_object.name, - mesh_detail=operator.mesh_detail, - plane_type=plane_type, - model=model, - resolution_level=resolution_level, - max_input_size=configured_max_ai_input_size(context), - delete_input=delete_input, - ) - if "CANCELLED" in result and delete_input: - cleanup_image_input(source_path, True) + setup_result = invoke_ai_setup_if_needed(operator) + if setup_result is not None: + return setup_result + color_path = None + try: + color_path = prepare_material_color_input(source_object.data) + source_path = material_analysis_input(color_path) + model, resolution_level = depth_plane_settings(context) + result = bpy.ops.anyimage.generate_depth_plane( + input_path=str(source_path), + color_path=str(color_path), + source_object_name=source_object.name, + source_identity=str(source_object.as_pointer()), + image_identity=str(source_object.data.as_pointer()), + mesh_detail=operator.mesh_detail, + plane_type=plane_type, + model=model, + resolution_level=resolution_level, + max_input_size=configured_max_ai_input_size(context), + ) + except (OSError, RuntimeError, ValueError) as error: + cleanup_material_color_input(color_path) + report_ai_error(operator, error) + return {"CANCELLED"} + if "CANCELLED" in result: + cleanup_material_color_input(color_path) return {"CANCELLED"} if "CANCELLED" in result else {"FINISHED"} @@ -143,20 +149,26 @@ class ConvertToPlane(bpy.types.Operator): def execute(self, context): source_object = context.object - color_image = create_color_image(source_object) - frame_start, frame_offset, frame_duration, use_cyclic = image_user_settings( - source_object - ) - material = create_image_material( - source_object.data, - color_image, - frame_start=frame_start, - color_frame_offset=frame_offset, - color_frame_duration=frame_duration, - use_cyclic=use_cyclic, - scene=context.scene, - ) - create_plane_object(context, source_object, self.mesh_detail, material) + try: + require_static_color_image(source_object.data) + except ValueError as error: + self.report({"ERROR"}, str(error)) + return {"CANCELLED"} + material = None + try: + with material_color_image(source_object.data) as color_image: + material = create_image_material( + source_object.data, color_image, + scene=context.scene, + ) + create_plane_object(context, source_object, self.mesh_detail, material) + except Exception as error: + if material is not None and material.users == 0: + bpy.data.materials.remove(material, do_unlink=True) + if isinstance(error, ValueError): + self.report({"ERROR"}, str(error)) + return {"CANCELLED"} + raise return {"FINISHED"} diff --git a/src/anyimage/operators/cutout_tool/object.py b/src/anyimage/operators/cutout_tool/object.py index 86cd703..571c4b9 100644 --- a/src/anyimage/operators/cutout_tool/object.py +++ b/src/anyimage/operators/cutout_tool/object.py @@ -4,7 +4,6 @@ from enum import StrEnum import bpy -from ...common.image import color_data_name, configure_color_alpha from ...common.depth import camera_depth_values, depth_uniform_scale from ...common.material import ( IMAGE_MATERIAL_NODE_GROUP_NAME, @@ -97,37 +96,6 @@ def _assign_float_point_attribute(mesh, name, values): attribute.data.foreach_set("value", np.asarray(values, dtype=np.float32)) -def create_cropped_color_image(source_object, crop_bounds, source_rgba): - """Create a packed color image for one local Cutout bounds rectangle.""" - import numpy as np - - source = source_object.data - rgba = np.asarray(source_rgba) - left, top, right, bottom = crop_bounds - width = right - left - height = bottom - top - if width <= 0 or height <= 0: - raise ValueError("The Cutout crop dimensions must be positive") - source_height, source_width = rgba.shape[:2] - if left < 0 or top < 0 or right > source_width or bottom > source_height: - raise ValueError("The Cutout crop is outside the source RGBA") - cropped = rgba[top:bottom, left:right] - image = bpy.data.images.new( - color_data_name(source_object), - width=width, - height=height, - alpha=True, - ) - try: - image.colorspace_settings.name = source.colorspace_settings.name - configure_color_alpha(image) - image.pixels.foreach_set(np.flipud(cropped).ravel()) - image.pack() - image.update() - except Exception: - bpy.data.images.remove(image, do_unlink=True) - raise - return image def load_cutout_node_group(shape): diff --git a/src/anyimage/operators/cutout_tool/operators.py b/src/anyimage/operators/cutout_tool/operators.py index ed0ad23..b993a1d 100644 --- a/src/anyimage/operators/cutout_tool/operators.py +++ b/src/anyimage/operators/cutout_tool/operators.py @@ -31,17 +31,19 @@ from ...common.ai import ( require_model, ) from ...common.image import ( - cleanup_image_input, - color_data_name, image_rgba, load_normal_result_image, - load_packed_rgba_image, - prepare_image_bounds_input, require_image_empty, - source_start_frame, video_total_frames, ) from ...common.depth import load_depth_metadata, load_depth_result_image +from ...common.color_image import ( + cleanup_material_color_input, + load_material_color_image, + material_analysis_input, + material_color_image, + prepare_material_color_input, +) from ...common.viewport import ( ImageGesture, image_edit_drag_keymap, @@ -61,7 +63,6 @@ from .interaction import open_shape_pie from .object import ( CutoutInwardAxis, NormalMode, - create_cropped_color_image, create_shape_object, orient_object_normal_image, ) @@ -101,36 +102,31 @@ def create_cutout_shape( inward_axis=CutoutInwardAxis.NEGATIVE_Z, color_image, ): - try: - base_shape = build_base_shape( - context, - source_object, - point_spacing, - content_values, - content_bounds, - alpha_threshold=alpha_threshold, - fine_outline=fine_outline, - ) - result = create_shape_object( - context, - source_object, - shape, - point_spacing, - content_values, - content_bounds, - base_shape, - alpha_threshold=alpha_threshold, - depth_image=depth_image, - depth_metadata=depth_metadata, - normal_image=normal_image, - normal_mode=normal_mode, - inward_axis=inward_axis, - color_image=color_image, - ) - except Exception: - if color_image.users == 0: - bpy.data.images.remove(color_image, do_unlink=True) - raise + base_shape = build_base_shape( + context, + source_object, + point_spacing, + content_values, + content_bounds, + alpha_threshold=alpha_threshold, + fine_outline=fine_outline, + ) + result = create_shape_object( + context, + source_object, + shape, + point_spacing, + content_values, + content_bounds, + base_shape, + alpha_threshold=alpha_threshold, + depth_image=depth_image, + depth_metadata=depth_metadata, + normal_image=normal_image, + normal_mode=normal_mode, + inward_axis=inward_axis, + color_image=color_image, + ) undo_message = f"Convert to {shape.replace('_', ' ').title()}" if "FINISHED" not in bpy.ops.ed.undo_push(message=undo_message): raise RuntimeError(f"Unable to create the {undo_message} Undo step") @@ -160,12 +156,8 @@ def _cutout_content_values( return values -def _generated_color_result(input_path, source_object, source_bounds): - image = load_packed_rgba_image( - input_path, - source_object.data, - color_data_name(source_object), - ) +def _generated_color_result(input_path, source_bounds): + image = load_material_color_image(input_path) try: alpha = image_rgba(image)[:, :, 3] except Exception: @@ -193,6 +185,7 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator): starting_message = "Generating Cutout artifacts" input_path: bpy.props.StringProperty(options=_HIDDEN) + color_path: bpy.props.StringProperty(options=_HIDDEN) source_object_name: bpy.props.StringProperty(options=_HIDDEN) selection_path_json: bpy.props.StringProperty(options=_HIDDEN) selection_bounds_json: bpy.props.StringProperty(options=_HIDDEN) @@ -265,25 +258,15 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator): selection_mask.values, alpha_threshold=self.alpha_threshold, ) - color_image = create_cropped_color_image( - source_object, - selection_mask.bounds, - source_rgba, - ) - create_cutout_shape( - context, - source_object, - self.shape, - self.point_spacing, - content_values, - selection_mask.bounds, - inward_axis=self.inward_axis, - alpha_threshold=self.alpha_threshold, - fine_outline=self.fine_outline, - color_image=color_image, - ) + with material_color_image(source_object.data, bounds=selection_mask.bounds, rgba=source_rgba) as color_image: + create_cutout_shape( + context, source_object, self.shape, self.point_spacing, + content_values, selection_mask.bounds, + inward_axis=self.inward_axis, alpha_threshold=self.alpha_threshold, + fine_outline=self.fine_outline, color_image=color_image, + ) return {"FINISHED"} - setup_result = invoke_ai_setup_if_needed() + setup_result = invoke_ai_setup_if_needed(self) if setup_result is not None: return setup_result self.model, self.resolution_level = cutout_geometry_settings(context) @@ -292,13 +275,10 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator): separators=(",", ":"), ) self._selection_mask = selection_mask - self.input_path = str( - prepare_image_bounds_input( - source_object.data, - source_rgba, - selection_mask.bounds, - ) - ) + self.color_path = str(prepare_material_color_input( + source_object.data, selection_mask.bounds, source_rgba, + )) + self.input_path = str(material_analysis_input(self.color_path)) except ( ImageEditWarning, OSError, @@ -306,11 +286,12 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator): TypeError, ValueError, ) as error: + cleanup_material_color_input(getattr(self, "color_path", None)) report_image_edit_exception(self, error) return {"CANCELLED"} result = super().execute(context) if "CANCELLED" in result: - cleanup_image_input(self.input_path, True) + cleanup_material_color_input(self.color_path) return result def request(self, context): @@ -329,7 +310,6 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator): "alpha_threshold": float(self.alpha_threshold), } if generate_depth or normal_mode != NormalMode.NONE: - start_frame = source_start_frame(self.source_object_name) parameters.update( model=self.model, resolution_level=self.resolution_level, @@ -340,7 +320,7 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator): source_path, self.model, self.resolution_level, - start_frame, + 1, ) ) return parameters @@ -354,14 +334,12 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator): source_object = require_image_empty(self.source_object_name) source_bounds = tuple(json.loads(self.selection_bounds_json)) color_image, source_alpha, color_bounds = _generated_color_result( - self.input_path, - source_object, + self.color_path, source_bounds, ) value = result.value if isinstance(result.value, dict) else {} normal_image = None depth_image = None - shape_started = False try: selection_mask = getattr(self, "_selection_mask", None) if selection_mask is None: @@ -389,7 +367,6 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator): source_object, depth_metadata, ) - shape_started = True create_cutout_shape( context, source_object, @@ -411,13 +388,13 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator): bpy.data.images.remove(normal_image, do_unlink=True) if depth_image is not None and depth_image.users == 0: bpy.data.images.remove(depth_image, do_unlink=True) - if not shape_started and color_image.users == 0: + if color_image.users == 0: bpy.data.images.remove(color_image, do_unlink=True) raise return f"{self.shape.replace('_', ' ').title()} is ready" def cleanup(self): - cleanup_image_input(self.input_path, True) + cleanup_material_color_input(self.color_path) class SelectCutoutSelection(ImageGesture, bpy.types.Operator): diff --git a/src/anyimage/operators/remove_background.py b/src/anyimage/operators/remove_background.py index 0146d34..3b593b0 100644 --- a/src/anyimage/operators/remove_background.py +++ b/src/anyimage/operators/remove_background.py @@ -3,7 +3,8 @@ import bpy from ..properties import BEN2_MODEL_KEY from ..runtime import JobOperatorBase from ..common.ai import ( - require_background_input_path, + require_input_path, + report_ai_error, invoke_ai_setup_if_needed, production_device, require_environment, @@ -12,6 +13,7 @@ from ..common.ai import ( from ..common.image import cleanup_image_input from ..common.image_target import ImageEditTarget, image_edit_owner +from ..common.hdr_image import HdrBackgroundInput _HIDDEN = {"HIDDEN", "SKIP_SAVE"} @@ -30,13 +32,17 @@ class RemoveImageBackground(bpy.types.Operator): return image_edit_owner(context) is not None and not runtime.server_busy() def invoke(self, _context, _event): - setup_result = invoke_ai_setup_if_needed() + setup_result = invoke_ai_setup_if_needed(self) if setup_result is not None: return setup_result return self.execute(_context) def execute(self, _context): - result = bpy.ops.anyimage.run_background_removal() + try: + result = bpy.ops.anyimage.run_background_removal() + except RuntimeError as error: + report_ai_error(self, error) + return {"CANCELLED"} if "CANCELLED" not in result: return {"FINISHED"} return {"CANCELLED"} @@ -52,14 +58,26 @@ class RunBackgroundRemoval(JobOperatorBase, bpy.types.Operator): delete_input: bpy.props.BoolProperty(default=False, options=_HIDDEN) def execute(self, context): + self._hdr_input = None try: self._image_target = ImageEditTarget.capture(context) - source_path, self.delete_input = self._image_target.prepare(context) - except RuntimeError as error: - self.report({"ERROR"}, str(error)) + if getattr(self._image_target.image, "is_float", False): + self._hdr_input = HdrBackgroundInput.prepare(self._image_target.image) + source_path, self.delete_input = self._hdr_input.path, True + else: + source_path, self.delete_input = self._image_target.prepare(context) + except (OSError, RuntimeError, ValueError, MemoryError) as error: + self._hdr_input = None + self._image_target = None + message = "Not enough memory to prepare the HDR image" if isinstance(error, MemoryError) else str(error) + self.report({"ERROR"}, message) return {"CANCELLED"} self.input_path = str(source_path) - result = super().execute(context) + try: + result = super().execute(context) + except Exception: + self.cleanup() + raise if "CANCELLED" in result: self.cleanup() return result @@ -68,20 +86,29 @@ class RunBackgroundRemoval(JobOperatorBase, bpy.types.Operator): require_environment() require_model(BEN2_MODEL_KEY) settings = context.scene.anyimage_settings - return { - "input": str(require_background_input_path(self.input_path)), + parameters = { + "input": str(require_input_path(self.input_path)), "model": BEN2_MODEL_KEY, "device": production_device().lower(), "video_frames": settings.video_frames, "start_frame": self._image_target.start_frame, } + if getattr(self, "_hdr_input", None) is not None: + parameters.update(output_kind="alpha", video_frames=1, start_frame=1) + return parameters def response(self, _context, result): - self._image_target.apply(result.directory) + if getattr(self, "_hdr_input", None) is not None: + self._hdr_input.apply(self._image_target, result.file("alpha"), prepare_undo=True) + else: + self._image_target.apply(result.directory) if "FINISHED" not in bpy.ops.ed.undo_push(message="Remove Background"): raise RuntimeError("Unable to create the Remove Background Undo step") return "Image background is removed" def cleanup(self): - cleanup_image_input(self.input_path, self.delete_input) - self._image_target = None + try: + cleanup_image_input(self.input_path, self.delete_input) + finally: + self._image_target = None + self._hdr_input = None diff --git a/src/anyimage/operators/upscale.py b/src/anyimage/operators/upscale.py index 37f822c..fe53a16 100644 --- a/src/anyimage/operators/upscale.py +++ b/src/anyimage/operators/upscale.py @@ -77,7 +77,7 @@ class UpscaleImage(JobOperatorBase, bpy.types.Operator): ) def invoke(self, context, _event): - setup_result = invoke_ai_setup_if_needed() + setup_result = invoke_ai_setup_if_needed(self) if setup_result is not None: return setup_result return self.execute(context) @@ -87,7 +87,7 @@ class UpscaleImage(JobOperatorBase, bpy.types.Operator): self.max_input_size = configured_max_ai_input_size(context) try: self._image_target = ImageEditTarget.capture(context) - except RuntimeError as error: + except (OSError, RuntimeError, ValueError) as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} source_image = self._image_target.image @@ -100,7 +100,7 @@ class UpscaleImage(JobOperatorBase, bpy.types.Operator): return {"CANCELLED"} try: source_path, delete_input = self._image_target.prepare(context) - except RuntimeError as error: + except (OSError, RuntimeError, ValueError) as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} diff --git a/src/anyimage/properties.py b/src/anyimage/properties.py index 9991fa8..7113243 100644 --- a/src/anyimage/properties.py +++ b/src/anyimage/properties.py @@ -139,7 +139,7 @@ def _model_items(models): def geometry_model_enum_items(_owner, _context): - models = [model for model in model_catalog() if model.get("family") == "moge2"] + models = [model for model in model_catalog() if model.get("family") == "moge"] return _model_items(models) diff --git a/src/anyimage/server/app.py b/src/anyimage/server/app.py index 9543340..379b707 100644 --- a/src/anyimage/server/app.py +++ b/src/anyimage/server/app.py @@ -4,6 +4,7 @@ from blendjob import JobServer from .jobs.cutout import run as _generate_cutout_artifacts from .jobs.depth_plane import run as _generate_depth_plane_geometry +from .jobs.panorama import run as _generate_panorama_geometry from .jobs.remove_background import run as _remove_background from .jobs.upscale import run as _upscale_image from .model_manager import ModelManager @@ -46,3 +47,8 @@ def generate_depth_plane_geometry(context, parameters): @server.job("generate-cutout-artifacts") def generate_cutout_artifacts(context, parameters): return _generate_cutout_artifacts(context, parameters) + + +@server.job("generate-panorama-geometry") +def generate_panorama_geometry(context, parameters): + return _generate_panorama_geometry(context, parameters) diff --git a/src/anyimage/server/geometry/depth_texture.py b/src/anyimage/server/geometry/depth_texture.py index 565d167..727d6f0 100644 --- a/src/anyimage/server/geometry/depth_texture.py +++ b/src/anyimage/server/geometry/depth_texture.py @@ -25,7 +25,7 @@ def _exr_channels(): ) + b"\0" -def _write_float_exr(rgba, output_path): +def write_float_exr(rgba, output_path): """Write one uncompressed scanline RGBA OpenEXR without extra dependencies.""" import numpy as np @@ -87,7 +87,7 @@ def write_depth_texture(frame, output_path, *, alpha): rgba = np.empty((*validity.shape, 4), dtype=np.float32) rgba[..., :3] = points rgba[..., 3] = alpha * validity - return _write_float_exr(rgba, output_path) + return write_float_exr(rgba, output_path) def write_depth_metadata(frame, reference_valid, output_path): diff --git a/src/anyimage/server/geometry/panorama.py b/src/anyimage/server/geometry/panorama.py new file mode 100644 index 0000000..debf611 --- /dev/null +++ b/src/anyimage/server/geometry/panorama.py @@ -0,0 +1,156 @@ +"""Spherical projection and periodic radial-distance fusion.""" + +import numpy as np +from scipy.ndimage import map_coordinates +from scipy.sparse import coo_matrix, diags +from scipy.sparse.linalg import spsolve + + +def panorama_directions(width, height): + u, v = np.meshgrid((np.arange(width) + 0.5) / width, + (np.arange(height) + 0.5) / height) + return uv_directions(u, v) + + +def uv_directions(u, v): + theta, phi = 2 * np.pi * u, np.pi * v + return np.stack((np.sin(phi) * np.cos(theta), + np.sin(phi) * np.sin(theta), np.cos(phi)), axis=-1) + + +def camera_bases(): + ratio = (1 + np.sqrt(5)) / 2 + vertices = [] + for a in (-1, 1): + for b in (-ratio, ratio): + vertices.extend(((0, a, b), (a, b, 0), (b, 0, a))) + bases = [] + for forward in np.asarray(vertices): + forward = forward / np.linalg.norm(forward) + right = np.cross(forward, (0, 0, 1)) + right /= np.linalg.norm(right) + down = np.cross(forward, right) + bases.append(np.stack((right, down, forward))) + return bases + + +def view_rays(size): + x, y = np.meshgrid(2 * (np.arange(size) + 0.5) / size - 1, + 2 * (np.arange(size) + 0.5) / size - 1) + return np.stack((x, y, np.ones_like(x)), axis=-1) + + +def sample_panorama(image, directions, *, order=1): + directions = directions / np.linalg.norm(directions, axis=-1, keepdims=True) + u = np.arctan2(directions[..., 1], directions[..., 0]) / (2 * np.pi) % 1 + v = np.arccos(np.clip(directions[..., 2], -1, 1)) / np.pi + height, width = image.shape[:2] + padded = np.concatenate((image[:, -1:], image, image[:, :1]), axis=1) + coords = (np.clip(v * height - 0.5, 0, height - 1), u * width + 0.5) + if image.ndim == 2: + return map_coordinates(padded, coords, order=order, mode="nearest") + return np.stack([map_coordinates(padded[..., c], coords, order=order, + mode="nearest") for c in range(image.shape[2])], -1) + + +def warp_distances(distances, masks, bases, width, height, *, directions=None): + if directions is None: + directions = panorama_directions(width, height) + logs, validities = [], [] + for distance, mask, basis in zip(distances, masks, bases): + rays = directions @ basis.T + projected = rays[..., :2] / np.maximum(rays[..., 2:], 1e-8) + valid = (rays[..., 2] > 0) & (np.abs(projected) < 0.98).all(axis=-1) + size = distance.shape[0] + coords = ((projected[..., 1] + 1) * size / 2 - 0.5, + (projected[..., 0] + 1) * size / 2 - 0.5) + valid &= map_coordinates(mask.astype(float), coords, order=0, mode="nearest") > 0.5 + logs.append(map_coordinates(np.log(np.maximum(distance, 1e-6)), coords, + order=1, mode="nearest")) + validities.append(valid) + return np.asarray(logs), np.asarray(validities) + + +def merge_distances(distances, masks, bases, width, height, *, cancel_check=None): + """Fuse log-distance derivatives with a cyclic longitude and a scale anchor. + + Algorithm reference: microsoft/MoGe, moge/utils/panorama.py. + This independent implementation solves normal equations on a bounded panorama grid. + """ + if cancel_check is not None: + cancel_check() + if width > 512: + return _refine_distances(distances, masks, bases, width, height, cancel_check) + logs, valid = warp_distances(distances, masks, bases, width, height) + view_count = len(logs) + count = width * height + index = np.arange(count).reshape(height, width) + start = np.concatenate((index.ravel(), index[:-1].ravel())) + end = np.concatenate((np.roll(index, -1, axis=1).ravel(), index[1:].ravel())) + rows = np.arange(len(start)) + gradient = coo_matrix((np.concatenate((-np.ones(len(start)), np.ones(len(start)))), + (np.tile(rows, 2), np.concatenate((start, end)))), + shape=(len(start), count)).tocsr() + laplacian = -(gradient.T @ gradient).tocsr() + edge_valid = valid.reshape(view_count, -1)[:, start] & valid.reshape(view_count, -1)[:, end] + grad_samples = logs.reshape(view_count, -1)[:, end] - logs.reshape(view_count, -1)[:, start] + grad_target = np.sum(grad_samples * edge_valid, axis=0) / np.maximum(edge_valid.sum(0), 1) + lap_valid = valid.copy() + lap_valid &= np.roll(valid, 1, axis=2) & np.roll(valid, -1, axis=2) + lap_valid[:, 1:] &= valid[:, :-1] + lap_valid[:, :-1] &= valid[:, 1:] + lap_valid = lap_valid.reshape(view_count, -1) + lap_samples = np.asarray((laplacian @ logs.reshape(view_count, -1).T).T) + lap_target = np.sum(lap_samples * lap_valid, axis=0) / np.maximum(lap_valid.sum(0), 1) + g = gradient[edge_valid.any(0)] + l = laplacian[lap_valid.any(0)] + coverage = valid.any(0) + if not coverage.any(): + raise ValueError("No valid panorama predictions") + baseline = np.sum(logs * valid, axis=0) / np.maximum(valid.sum(0), 1) + matrix = g.T @ g + l.T @ l + diags(np.full(count, 1e-7)) + target = g.T @ grad_target[edge_valid.any(0)] + l.T @ lap_target[lap_valid.any(0)] + if cancel_check is not None: + cancel_check() + solution = spsolve(matrix.tocsc(), target + 1e-7 * baseline.ravel()).reshape(height, width) + if cancel_check is not None: + cancel_check() + solution += np.median(baseline[coverage] - solution[coverage]) + return np.exp(solution).astype(np.float32), coverage + + +def _refine_distances(distances, masks, bases, width, height, cancel_check): + """Apply coarse periodic alignment to view detail in bounded output tiles.""" + coarse_width, coarse_height = 512, 256 + coarse, _ = merge_distances(distances, masks, bases, coarse_width, coarse_height, + cancel_check=cancel_check) + logs, valid = warp_distances(distances, masks, bases, coarse_width, coarse_height) + corrections = np.log(coarse)[None] - logs + result = np.empty((height, width), np.float32) + mask = np.empty((height, width), bool) + tile_height = max(1, 65536 // width) + for start in range(0, height, tile_height): + if cancel_check is not None: + cancel_check() + stop = min(start + tile_height, height) + u, v = np.meshgrid((np.arange(width) + 0.5) / width, + (np.arange(start, stop) + 0.5) / height) + directions = uv_directions(u, v) + detail, visible = warp_distances(distances, masks, bases, width, height, + directions=directions) + total = np.zeros(u.shape) + weights = np.zeros(u.shape) + for index, basis in enumerate(bases): + support = sample_panorama(valid[index].astype(float), directions) + correction = sample_panorama(corrections[index] * valid[index], directions) + correction /= np.maximum(support, 1e-8) + forward = directions @ basis[2] + weight = visible[index] * np.maximum(forward, 0) ** 4 * (support > 1e-8) + total += (detail[index] + correction) * weight + weights += weight + fallback = sample_panorama(np.log(coarse), directions) + np.divide(total, weights, out=fallback, where=weights > 0) + result[start:stop] = np.exp(fallback) + mask[start:stop] = visible.any(axis=0) + return result, mask + diff --git a/src/anyimage/server/geometry/prediction_artifacts.py b/src/anyimage/server/geometry/prediction_artifacts.py index 833bc85..ae04ea1 100644 --- a/src/anyimage/server/geometry/prediction_artifacts.py +++ b/src/anyimage/server/geometry/prediction_artifacts.py @@ -1,7 +1,8 @@ -"""Export requested depth and normal artifacts from one MoGe-2 prediction.""" +"""Export requested depth and normal artifacts from one MoGe prediction.""" +from ..media.input import open_image from ..media import DEFAULT_MAX_AI_INPUT_SIZE, limit_image -from ..models import moge2 +from ..models import moge from .depth_texture import ( write_depth_metadata, write_depth_texture, @@ -11,9 +12,7 @@ from .edge_depth import extend_edge_depth def _limited_model_input(source_path, output_path, max_input_size): - from PIL import Image - - with Image.open(source_path) as opened: + with open_image(source_path) as opened: limited = limit_image(opened.convert("RGB"), max_input_size) limited.save(output_path, format="PNG", compress_level=1) return output_path @@ -23,7 +22,7 @@ def _image_alpha(image_path, shape): import numpy as np from PIL import Image - with Image.open(image_path) as opened: + with open_image(image_path) as opened: alpha = opened.convert("RGBA").getchannel("A") alpha = alpha.resize( (int(shape[1]), int(shape[0])), @@ -33,10 +32,8 @@ def _image_alpha(image_path, shape): def _write_normal(frame, color_path, output_path, normal_mode): - from PIL import Image - valid = _image_alpha(color_path, frame.depth.shape) >= 0.1 - with Image.open(color_path) as source: + with open_image(color_path) as source: output_size = source.size return write_camera_normal_texture( frame, @@ -49,7 +46,7 @@ def _write_normal(frame, color_path, output_path, normal_mode): ) -def generate_moge2_artifacts( +def generate_moge_artifacts( context, parameters, input_path, @@ -58,7 +55,7 @@ def generate_moge2_artifacts( normal_mode="NONE", alpha_threshold=0.9, ): - """Run MoGe-2 once and write only the requested file artifacts.""" + """Run MoGe once and write only the requested file artifacts.""" normal_mode = str(normal_mode).upper() if normal_mode not in {"NONE", "TANGENT", "OBJECT"}: raise ValueError(f"Unsupported normal mode: {normal_mode}") @@ -68,7 +65,7 @@ def generate_moge2_artifacts( parameters.get("max_input_size", DEFAULT_MAX_AI_INPUT_SIZE), ) try: - frames = moge2.infer( + frames = moge.infer( context.resource("model_manager"), parameters, [model_path], diff --git a/src/anyimage/server/jobs/cutout.py b/src/anyimage/server/jobs/cutout.py index 0ff18e3..f52ccb0 100644 --- a/src/anyimage/server/jobs/cutout.py +++ b/src/anyimage/server/jobs/cutout.py @@ -2,7 +2,7 @@ from pathlib import Path -from ..geometry.prediction_artifacts import generate_moge2_artifacts +from ..geometry.prediction_artifacts import generate_moge_artifacts def run(context, parameters): @@ -13,7 +13,7 @@ def run(context, parameters): generate_depth = bool(parameters.get("generate_depth", False)) normal_mode = str(parameters.get("normal_mode", "NONE")) if generate_depth or normal_mode != "NONE": - artifacts = generate_moge2_artifacts( + artifacts = generate_moge_artifacts( context, parameters, input_path, diff --git a/src/anyimage/server/jobs/depth_plane.py b/src/anyimage/server/jobs/depth_plane.py index 6e85dd9..8078605 100644 --- a/src/anyimage/server/jobs/depth_plane.py +++ b/src/anyimage/server/jobs/depth_plane.py @@ -1,7 +1,7 @@ -"""Generate model-space depth and preserve color for one complete image.""" +"""Generate model-space depth and normals for one complete image.""" -from ..geometry.prediction_artifacts import generate_moge2_artifacts -from ..media import copy_color_images, input_frames +from ..geometry.prediction_artifacts import generate_moge_artifacts +from ..media import input_frames def run(context, parameters): @@ -13,8 +13,7 @@ def run(context, parameters): with input_frames(context, parameters) as images: if len(images) != 1: raise ValueError("Depth conversion currently supports one image") - copy_color_images(images, context.directory) - artifacts = generate_moge2_artifacts( + artifacts = generate_moge_artifacts( context, parameters, images[0], diff --git a/src/anyimage/server/jobs/panorama.py b/src/anyimage/server/jobs/panorama.py new file mode 100644 index 0000000..e3b3ae4 --- /dev/null +++ b/src/anyimage/server/jobs/panorama.py @@ -0,0 +1,82 @@ +"""Generate a radial depth texture from the current RGBA panorama.""" + +import json +from pathlib import Path + +from ..geometry.depth_texture import write_float_exr +from ..geometry.panorama import camera_bases, merge_distances, sample_panorama, view_rays +from ..media import VIDEO_SUFFIXES, input_frames +from ..media.input import open_image +from ..media.resolution import DEFAULT_MAX_AI_INPUT_SIZE, normalized_max_input_size +from ..models.moge import infer_image as infer + + +def run(context, parameters): + import numpy as np + + context.check_cancelled() + input_path = Path(parameters["input"]) + if input_path.is_dir() or input_path.suffix.lower() in VIDEO_SUFFIXES: + raise ValueError("Panorama conversion requires one static 2:1 image") + maximum = normalized_max_input_size(parameters.get("max_input_size", DEFAULT_MAX_AI_INPUT_SIZE)) + context.directory.mkdir(parents=True, exist_ok=True) + outputs = [context.directory / name for name in ("depth.exr", "depth.json")] + try: + with input_frames(context, parameters) as images: + if len(images) != 1: + raise ValueError("Panorama conversion requires one static 2:1 image") + with open_image(images[0]) as opened: + if getattr(opened, "n_frames", 1) != 1 or opened.width != opened.height * 2: + raise ValueError("Panorama conversion requires one static 2:1 image") + source = opened.convert("RGBA") + width, height = source.size + size = min(maximum, max(512, width // 4)) + image = np.asarray(source, dtype=np.float32) / 255.0 + manager = context.resource("model_manager") + bases = camera_bases() + rays = view_rays(size) + distances, masks = [], [] + session = None + for index, basis in enumerate(bases): + context.check_cancelled() + context.progress(0.05 + 0.65 * index / len(bases), f"Generating panorama view {index + 1}/12") + directions = rays @ basis + alpha = sample_panorama(image[..., 3], directions, order=0) + visible = alpha > 0 + if not visible.any(): + distances.append(np.ones((size, size), dtype=np.float32)) + masks.append(visible) + continue + if session is None: + session = manager.get_moge(manager.directory(parameters["model"]), parameters["device"]) + color = sample_panorama(image[..., :3], directions) * alpha[..., None] + prediction = infer(session, color, parameters["resolution_level"], + fov_x=90.0, source_valid=visible) + distances.append(np.linalg.norm(prediction["points"], axis=-1)) + masks.append((prediction["mask"] > 0.5) & visible) + del prediction + context.progress(0.75, "Merging panoramic distance") + distance, valid = merge_distances(distances, masks, bases, width, height, + cancel_check=context.check_cancelled) + alpha = np.asarray(source.getchannel("A"), + dtype=np.float32) / 255.0 + alpha *= valid + reference = (alpha >= 0.9) & np.isfinite(distance) & (distance > 0) + if not reference.any(): + raise ValueError("The panorama contains no valid depth pixels") + if not np.isfinite(distance).all() or (distance <= 0).any(): + raise ValueError("Panorama fusion produced invalid distances") + rgba = np.empty((height, width, 4), dtype=np.float32) + rgba[..., :3], rgba[..., 3] = distance[..., None], alpha + write_float_exr(rgba, outputs[0]) + outputs[1].write_text(json.dumps({ + "projection": "equirectangular", "image_size": [width, height], + "reference_distance": float(np.median(distance[reference])), + }, separators=(",", ":")), encoding="utf-8") + context.check_cancelled() + context.progress(1.0, "Panorama conversion data is ready") + return {"depth": outputs[0].name, "depth_metadata": outputs[1].name} + except BaseException: + for path in outputs: + path.unlink(missing_ok=True) + raise diff --git a/src/anyimage/server/jobs/remove_background.py b/src/anyimage/server/jobs/remove_background.py index 49c68e9..908582c 100644 --- a/src/anyimage/server/jobs/remove_background.py +++ b/src/anyimage/server/jobs/remove_background.py @@ -1,7 +1,8 @@ import time from ..media import input_frames -from ..models.ben2 import infer_foreground, infer_foreground_frames +from ..media.input import open_image +from ..models.ben2 import infer_alpha, infer_foreground, infer_foreground_frames def run(context, parameters): @@ -15,10 +16,25 @@ def run(context, parameters): device = parameters["device"] model_manager = context.resource("model_manager") model_directory = model_manager.directory(parameters["model"]) + output_kind = parameters.get("output_kind", "foreground") + if output_kind not in {"foreground", "alpha"}: + raise ValueError(f"Unknown background removal output: {output_kind}") with input_frames(context, parameters) as images: context.progress(0.12, f"Loading BEN2 for {device.upper()}") - if len(images) == 1: + if output_kind == "alpha": + import numpy as np + + if len(images) != 1: + raise ValueError("Alpha output requires one still image") + with open_image(images[0]) as source: + alpha, load_ms, inference_ms = infer_alpha( + source, model_directory, device, model_cache=model_manager, + cancel_check=context.check_cancelled, + ) + context.check_cancelled() + np.save(output_directory / "alpha.npy", alpha, allow_pickle=False) + elif len(images) == 1: context.progress(0.35, "Removing background") foreground, load_ms, inference_ms = infer_foreground( images[0], @@ -62,4 +78,4 @@ def run(context, parameters): ) context.log(message) context.progress(1.0, message) - return {"image": "."} + return {"alpha": "alpha.npy"} if output_kind == "alpha" else {"image": "."} diff --git a/src/anyimage/server/media/__init__.py b/src/anyimage/server/media/__init__.py index a2dc6f8..f52df17 100644 --- a/src/anyimage/server/media/__init__.py +++ b/src/anyimage/server/media/__init__.py @@ -5,7 +5,6 @@ from .input import ( analysis_frames, check_cancelled, collect_images, - copy_color_images, extract_video_frames, first_frames, input_frames, @@ -29,7 +28,6 @@ __all__ = ( "analysis_frames", "check_cancelled", "collect_images", - "copy_color_images", "extract_video_frames", "first_frames", "input_frames", diff --git a/src/anyimage/server/media/input.py b/src/anyimage/server/media/input.py index 00ea478..043b733 100644 --- a/src/anyimage/server/media/input.py +++ b/src/anyimage/server/media/input.py @@ -16,6 +16,52 @@ VIDEO_SUFFIXES = { ".mpg", ".mpeg", } + + +def validate_input_path(value, *, images_only=False): + """Validate supported media without importing image or video libraries.""" + path = Path(value) + if path.is_dir() and not images_only: + if not collect_images(path): + supported = ", ".join(sorted(IMAGE_SUFFIXES)) + raise RuntimeError( + f"No supported image frames found in: {path}. Supported images: {supported}." + ) + return path + if not path.is_file(): + kind = "video" if path.suffix.lower() in VIDEO_SUFFIXES else "image" + raise RuntimeError(f"Source {kind} file does not exist: {path}") + suffix = path.suffix.lower() + supported = IMAGE_SUFFIXES if images_only else IMAGE_SUFFIXES | VIDEO_SUFFIXES + if suffix not in supported: + message = ( + f"Unsupported input format: {suffix or '(no extension)'}. " + f"Supported images: {', '.join(sorted(IMAGE_SUFFIXES))}." + ) + if not images_only: + message += f" Supported videos: {', '.join(sorted(VIDEO_SUFFIXES))}." + raise RuntimeError(message) + return path + + +@contextmanager +def open_image(path): + """Decode an image before handing it to processing code.""" + from PIL import Image + + opened = None + try: + try: + opened = Image.open(path) + opened.load() + except (OSError, ValueError, SyntaxError) as error: + raise RuntimeError(f"Unable to read image: {path}") from error + yield opened + finally: + if opened is not None: + opened.close() + + class TaskCancelled(RuntimeError): pass @@ -55,18 +101,32 @@ def extract_video_frames( try: container = av.open(str(video_path)) except Exception as error: - raise RuntimeError(f"Unable to open video: {video_path}") from error + raise RuntimeError(f"Unable to read video: {video_path}") from error image_paths = [] skipped = 0 try: - stream = container.streams.video[0] - for frame in container.decode(stream): + try: + stream = container.streams.video[0] + frames = iter(container.decode(stream)) + except Exception as error: + raise RuntimeError(f"Unable to read video: {video_path}") from error + while True: + try: + frame = next(frames) + except StopIteration: + break + except Exception as error: + raise RuntimeError(f"Unable to read video: {video_path}") from error check_cancelled(cancel_check) if skipped < start_frame - 1: skipped += 1 continue image_path = output_directory / f"frame.{len(image_paths) + 1:04d}.jpg" - frame.to_image().save(image_path, format="JPEG") + try: + decoded = frame.to_image() + except Exception as error: + raise RuntimeError(f"Unable to read video: {video_path}") from error + decoded.save(image_path, format="JPEG") image_paths.append(image_path) if len(image_paths) >= maximum_frames: break @@ -74,7 +134,7 @@ def extract_video_frames( container.close() if not image_paths: - raise RuntimeError(f"Unable to read any video frame from: {video_path}") + raise RuntimeError(f"Unable to read video: {video_path}") return image_paths @@ -86,18 +146,13 @@ def prepare_input( cancel_check=None, ): check_cancelled(cancel_check) - input_path = Path(input_path) + input_path = validate_input_path(input_path) if input_path.is_dir(): images = collect_images(input_path) - if not images: - raise RuntimeError(f"No image frames found in: {input_path}") return first_frames(images, video_frames, start_frame), None suffix = input_path.suffix.lower() if suffix in IMAGE_SUFFIXES: return [input_path], None - if suffix not in VIDEO_SUFFIXES: - supported = ", ".join(sorted(IMAGE_SUFFIXES | VIDEO_SUFFIXES)) - raise RuntimeError(f"Unsupported input type; expected one of: {supported}") import av @@ -145,15 +200,3 @@ def analysis_frames(context, parameters, fallback_images): return with input_frames(context, dict(parameters, input=analysis_input)) as images: yield images - - -def copy_color_images(images, output_directory): - """Copy input images as the canonical numbered image layer.""" - for index, image_path in enumerate(images): - suffix = Path(image_path).suffix.lower() or ".jpg" - destination = ( - Path(output_directory) / f"image.{index + 1:04d}{suffix}" - ) - if Path(image_path).resolve() == destination.resolve(): - continue - shutil.copy2(image_path, destination) diff --git a/src/anyimage/server/model_catalog.py b/src/anyimage/server/model_catalog.py index 04f55ef..67e6d90 100644 --- a/src/anyimage/server/model_catalog.py +++ b/src/anyimage/server/model_catalog.py @@ -178,10 +178,28 @@ MOGE2_MODELS = { ),), ), } +MOGE3_VITL = ModelSpec( + key="MOGE3_VITL", + label="MoGe-3 ViT-L", + directory_name="moge-3-vitl-onnx", + repository="Ruicheng/moge-3-vitl", + license_name="MIT", + description="MoGe-3 geometry with three refinement steps using ONNX Runtime", + download_patterns=("backbone.onnx", "refiner.onnx"), + ready_patterns=("backbone.onnx", "refiner.onnx"), + r2_directory="moge-3-vitl-onnx", + r2_files=( + ("backbone.onnx", 1324190318, "f21246c6921373c8c79ecdb8c78818a79bd2506c2fb88f4d4cdec21b92f725e3"), + ("refiner.onnx", 157729628, "67ad7297aba18f8ef7939e8f700a5b25c41a19ddb68656bc2cd08b54c248fe2a"), + ), +) + +MOGE_MODELS = {**MOGE2_MODELS, MOGE3_VITL.key: MOGE3_VITL} + DOWNLOADABLE_MODELS = { BEN2_MODEL.key: BEN2_MODEL, **UPSCALE_MODELS, - **MOGE2_MODELS, + **MOGE_MODELS, } @@ -200,8 +218,8 @@ def model_record(model, ready=False): "description": model.description, "license": model.license_name, "family": ( - "moge2" - if model.key.startswith("MOGE2_") + "moge" + if model.key in MOGE_MODELS else "ben2" if model.key == BEN2_MODEL.key else "upscale" diff --git a/src/anyimage/server/model_manager.py b/src/anyimage/server/model_manager.py index e8f5dc2..f3d5d88 100644 --- a/src/anyimage/server/model_manager.py +++ b/src/anyimage/server/model_manager.py @@ -38,9 +38,12 @@ class ModelSessionCache: return model, elapsed_ms - def get_moge2(self, model_dir, requested_device): + def get_moge(self, model_dir, requested_device): from .models.onnx_moge2 import create_session + if Path(model_dir).name == "moge-3-vitl-onnx": + from .models.onnx_moge3 import create_session + key = (str(Path(model_dir).resolve()), requested_device, "onnx") if self.moge_model is not None and self.moge_key == key: return self.moge_model @@ -122,7 +125,7 @@ class ModelManager: def ready(self, model_key): model = model_catalog.get_downloadable_model(model_key) directory = self.directory(model_key) - if model_key == "PISA_SR": + if model_key in {"PISA_SR", "MOGE3_VITL"}: for filename, size, checksum in model.r2_files: path = directory / filename try: @@ -149,8 +152,8 @@ class ModelManager: return self.sessions.get_ben2(model_dir, device) - def get_moge2(self, model_dir, device): - return self.sessions.get_moge2(model_dir, device) + def get_moge(self, model_dir, device): + return self.sessions.get_moge(model_dir, device) def get_upscale(self, model_dir, device): return self.sessions.get_upscale(model_dir, device) diff --git a/src/anyimage/server/models/__init__.py b/src/anyimage/server/models/__init__.py index bc25747..ebc8c63 100644 --- a/src/anyimage/server/models/__init__.py +++ b/src/anyimage/server/models/__init__.py @@ -1,7 +1,7 @@ -from . import ben2, moge2, upscale +from . import ben2, moge, upscale __all__ = ( "ben2", - "moge2", + "moge", "upscale", ) diff --git a/src/anyimage/server/models/ben2.py b/src/anyimage/server/models/ben2.py index b798bdc..6c74acb 100644 --- a/src/anyimage/server/models/ben2.py +++ b/src/anyimage/server/models/ben2.py @@ -1,25 +1,22 @@ import time from pathlib import Path -from ..media import IMAGE_SUFFIXES, check_cancelled +from ..media import check_cancelled +from ..media.input import open_image, validate_input_path -def infer_foreground( - input_path, +def infer_alpha( + image, model_dir, device, model_cache=None, cancel_check=None, - preserve_input_alpha=False, release_memory=True, ): - from PIL import Image, ImageChops + from PIL import Image - input_path = Path(input_path) check_cancelled(cancel_check) - if input_path.suffix.lower() not in IMAGE_SUFFIXES: - raise RuntimeError("BEN2 requires a JPG, JPEG, PNG, or WEBP image") - from .onnx_ben2 import create_session, infer + from .onnx_ben2 import create_session, infer_alpha as predict_alpha owns_model = model_cache is None requested_device = getattr(device, "type", device) @@ -31,8 +28,7 @@ def infer_foreground( model, load_ms = model_cache.get_ben2(model_dir, device) check_cancelled(cancel_check) - with Image.open(input_path) as opened_image: - source_rgba = opened_image.convert("RGBA") + source_rgba = image.convert("RGBA") image = Image.new("RGB", source_rgba.size, (0, 0, 0)) image.paste( source_rgba.convert("RGB"), @@ -40,23 +36,44 @@ def infer_foreground( source_rgba.getchannel("A"), ) inference_started = time.perf_counter() - foreground = infer( + alpha = predict_alpha( model, image, release_memory=release_memory, ) inference_ms = (time.perf_counter() - inference_started) * 1000.0 - if preserve_input_alpha: - foreground_alpha = foreground.convert("RGBA").getchannel("A") - source_alpha = source_rgba.getchannel("A") - if source_alpha.size != foreground_alpha.size: - source_alpha = source_alpha.resize( - foreground_alpha.size, - Image.Resampling.BILINEAR, - ) - source_rgba.putalpha(ImageChops.multiply(source_alpha, foreground_alpha)) - foreground = source_rgba check_cancelled(cancel_check) + return alpha, load_ms, inference_ms + + +def infer_foreground( + input_path, + model_dir, + device, + model_cache=None, + cancel_check=None, + preserve_input_alpha=False, + release_memory=True, +): + import numpy as np + from PIL import Image, ImageChops + + check_cancelled(cancel_check) + input_path = validate_input_path(Path(input_path), images_only=True) + with open_image(input_path) as opened_image: + source = opened_image.convert("RGBA") + alpha, load_ms, inference_ms = infer_alpha( + source, model_dir, device, model_cache=model_cache, + cancel_check=cancel_check, release_memory=release_memory, + ) + mask = Image.fromarray((alpha * 255).astype(np.uint8)) + if preserve_input_alpha: + foreground = source + mask = ImageChops.multiply(source.getchannel("A"), mask) + else: + foreground = Image.new("RGBA", source.size, (0, 0, 0, 255)) + foreground.paste(source.convert("RGB"), (0, 0), source.getchannel("A")) + foreground.putalpha(mask) return foreground, load_ms, inference_ms diff --git a/src/anyimage/server/models/moge2.py b/src/anyimage/server/models/moge.py similarity index 81% rename from src/anyimage/server/models/moge2.py rename to src/anyimage/server/models/moge.py index 9414141..485c54e 100644 --- a/src/anyimage/server/models/moge2.py +++ b/src/anyimage/server/models/moge.py @@ -1,3 +1,4 @@ +from ..media.input import open_image from .geometry import GeometryFrame @@ -71,20 +72,25 @@ def infer_one( release_memory=True, ): import numpy as np - from PIL import Image - - from .onnx_moge2 import infer as infer_onnx model_directory = model_manager.directory(parameters["model"]) - model = model_manager.get_moge2( + model = model_manager.get_moge( model_directory, parameters["device"], ) - with Image.open(image_path) as opened: + with open_image(image_path) as opened: image = np.asarray(opened.convert("RGB"), dtype=np.float32) / 255.0 - return infer_onnx( + return infer_image( model, image, int(parameters["resolution_level"]), release_memory=release_memory, ) + + +def infer_image(session, image, resolution_level, **options): + """Dispatch one RGB array to the selected MoGe ONNX model.""" + from . import onnx_moge2, onnx_moge3 + + backend = onnx_moge3 if isinstance(session, onnx_moge3.Moge3Session) else onnx_moge2 + return backend.infer(session, image, resolution_level, **options) diff --git a/src/anyimage/server/models/onnx_ben2.py b/src/anyimage/server/models/onnx_ben2.py index e458598..9ad038b 100644 --- a/src/anyimage/server/models/onnx_ben2.py +++ b/src/anyimage/server/models/onnx_ben2.py @@ -34,7 +34,7 @@ def preprocess(image): return np.ascontiguousarray(array.transpose(2, 0, 1)[None]) -def infer(session, image, *, release_memory=True): +def infer_alpha(session, image, *, release_memory=True): import numpy as np from PIL import Image @@ -46,12 +46,14 @@ def infer(session, image, *, release_memory=True): {input_name: preprocess(image)}, release_memory=release_memory, )[0] - alpha = np.asarray(alpha, dtype=np.float32).squeeze() + alpha = np.asarray(alpha, dtype=np.float32) + if (alpha.ndim != 4 or alpha.shape[:2] != (1, 1) + or not all(alpha.shape[2:]) or not np.isfinite(alpha).all()): + raise RuntimeError("BEN2 returned an invalid alpha mask") + alpha = alpha[0, 0] minimum = float(alpha.min()) span = max(float(alpha.max()) - minimum, np.finfo(np.float32).eps) - alpha = np.clip((alpha - minimum) / span * 255.0, 0.0, 255.0) - mask = Image.fromarray(alpha.astype(np.uint8), mode="L") + alpha = np.clip((alpha - minimum) / span, 0.0, 1.0) + mask = Image.fromarray(alpha) mask = mask.resize(image.size, Image.Resampling.LANCZOS) - foreground = image.convert("RGBA") - foreground.putalpha(mask) - return foreground + return np.clip(np.asarray(mask, dtype=np.float32), 0.0, 1.0) diff --git a/src/anyimage/server/models/onnx_moge2.py b/src/anyimage/server/models/onnx_moge2.py index 1759331..9c84ed5 100644 --- a/src/anyimage/server/models/onnx_moge2.py +++ b/src/anyimage/server/models/onnx_moge2.py @@ -109,7 +109,33 @@ def depth_to_points(depth, intrinsics): return np.stack((x, y, depth), axis=-1) -def postprocess(raw_output): +def recover_known_fov_shift(points, mask, fx, fy): + """Fit camera Z translation with fixed intrinsics on visible predictions.""" + import numpy as np + from scipy.optimize import least_squares + + height, width = mask.shape + intrinsics = np.array(((fx, 0, 0.5), (0, fy, 0.5), (0, 0, 1))) + rays = depth_to_points(np.ones((height, width)), intrinsics) + valid = (mask > 0.5) & np.isfinite(points).all(-1) + if valid.sum() < 16: + return 0.0, np.zeros_like(valid) + stride = max(1, min(height, width) // 64) + selected = valid[::stride, ::stride] + p = points[::stride, ::stride][selected].astype(float) + q = rays[::stride, ::stride, :2][selected] + if len(p) < 2: + p, q = points[valid].astype(float), rays[..., :2][valid] + initial = np.sum(q * (p[:, :2] - q * p[:, 2:])) / max(np.sum(q * q), 1e-8) + lower = -float(points[..., 2][valid].min()) + 1e-5 + fitted = least_squares( + lambda shift: (p[:, :2] / (p[:, 2:] + shift[0]) - q).ravel(), + [max(initial, lower + 1e-4)], bounds=([lower], [np.inf]), + ) + return fitted.x[0], valid + + +def postprocess(raw_output, *, fov_x=None, source_valid=None): import numpy as np points = np.asarray(raw_output["points"][0], dtype=np.float32) @@ -118,16 +144,30 @@ def postprocess(raw_output): metric_scale = np.float32(np.asarray(raw_output["metric_scale"]).reshape(-1)[0]) height, width = mask.shape aspect_ratio = width / height - focal, shift = recover_focal_shift(points, mask > 0.5) - diagonal_factor = (1.0 + aspect_ratio**2) ** 0.5 - fx = focal / 2.0 * diagonal_factor / aspect_ratio - fy = focal / 2.0 * diagonal_factor + if fov_x is None: + focal, shift = recover_focal_shift(points, mask > 0.5) + diagonal_factor = (1.0 + aspect_ratio**2) ** 0.5 + fx = focal / 2.0 * diagonal_factor / aspect_ratio + fy = focal / 2.0 * diagonal_factor + else: + if not 0.0 < fov_x < 180.0: + raise ValueError("Horizontal FOV must be between 0 and 180 degrees") + fx = 0.5 / np.tan(np.deg2rad(fov_x) / 2) + fy = fx * aspect_ratio + if source_valid is not None: + mask = mask * np.asarray(source_valid, dtype=bool) + shift, valid = recover_known_fov_shift(points, mask, fx, fy) + mask = np.where(valid, mask, 0.0) intrinsics = np.array( ((fx, 0.0, 0.5), (0.0, fy, 0.5), (0.0, 0.0, 1.0)), dtype=np.float32, ) depth = points[..., 2] + shift depth = depth * metric_scale + if fov_x is not None: + valid = np.isfinite(depth) & (depth > 0) + mask = np.where(valid, mask, 0.0) + depth = np.where(valid, depth, 1.0) points = depth_to_points(depth, intrinsics) return { "points": points.astype(np.float32), @@ -138,7 +178,8 @@ def postprocess(raw_output): } -def infer(session, image, resolution_level, *, release_memory=True): +def infer(session, image, resolution_level, *, release_memory=True, + fov_x=None, source_valid=None): import numpy as np level = min(max(int(resolution_level), 0), 9) @@ -159,4 +200,4 @@ def infer(session, image, resolution_level, *, release_memory=True): ) raw_output = dict(zip(names, values)) raw_output["metric_scale"] = raw_output.pop(scale_name) - return postprocess(raw_output) + return postprocess(raw_output, fov_x=fov_x, source_valid=source_valid) diff --git a/src/anyimage/server/models/onnx_moge3.py b/src/anyimage/server/models/onnx_moge3.py new file mode 100644 index 0000000..c10925f --- /dev/null +++ b/src/anyimage/server/models/onnx_moge3.py @@ -0,0 +1,159 @@ +"""MoGe-3 ONNX backbone and sparse refinement inference.""" + +from dataclasses import dataclass +from itertools import product +from pathlib import Path + +from .onnx_moge2 import MAX_TOKENS, MIN_TOKENS, postprocess +from .onnx_runtime import create_session as create_runtime_session +from .onnx_runtime import run_session + + +@dataclass +class Moge3Session: + backbone: object + refiner: object + + +def create_session(model_directory, requested_device="auto"): + directory = Path(model_directory) + backbone = create_runtime_session(directory / "backbone.onnx", requested_device) + refiner = create_runtime_session(directory / "refiner.onnx", requested_device) + return Moge3Session(backbone, refiner) + + +def build_sparse_inputs(coord): + """Build dynamic convolution, pooling and upsampling indices for five levels.""" + import numpy as np + + height, width = coord.shape[1:3] + z = np.rint(coord[0, ..., 2] * 256).astype(np.int64) + z -= z.min() + y, x = np.indices((height, width)) + coords = np.stack((y, x, z), axis=-1).reshape(-1, 3) + neighbors, pools, parents = [], [], [] + for level in range(5): + maximum = coords.max(axis=0) + dimensions = maximum + 3 + + def keys(values): + return ( + ((values[:, 0] + 1) * dimensions[1] + values[:, 1] + 1) * dimensions[2] + + values[:, 2] + + 1 + ) + + order = np.argsort(keys(coords)) + sorted_keys = keys(coords)[order] + indices = [] + for offset in product((-1, 0, 1), repeat=3): + query = coords + offset + query_keys = keys(query) + positions = np.minimum( + np.searchsorted(sorted_keys, query_keys), len(coords) - 1 + ) + valid = ( + (sorted_keys[positions] == query_keys) + & (query >= 0).all(axis=1) + & (query <= maximum).all(axis=1) + ) + indices.append(np.where(valid, order[positions], len(coords))) + neighbors.append(np.stack(indices, axis=1).astype(np.int64)) + if level < 4: + coarse, inverse = np.unique(coords // 2, axis=0, return_inverse=True) + children = np.full((len(coarse), 8), len(coords), dtype=np.int64) + slot = (coords[:, 0] % 2) * 4 + (coords[:, 1] % 2) * 2 + coords[:, 2] % 2 + children[inverse, slot] = np.arange(len(coords)) + pools.append(children) + parents.append(inverse.astype(np.int64)) + coords = coarse + encoder_index = (coords[:, 0] * (width // 16) + coords[:, 1]).astype(np.int64) + return { + **{f"neighbor_{i}": value for i, value in enumerate(neighbors)}, + **{f"pool_{i}": value for i, value in enumerate(pools)}, + **{f"up_{i}": value for i, value in enumerate(parents)}, + "encoder_index": encoder_index, + } + + +def _resize(array, height, width): + """Resize an HWC array with half-pixel bilinear sampling.""" + import numpy as np + from scipy.ndimage import map_coordinates + + y = (np.arange(height) + 0.5) * array.shape[0] / height - 0.5 + x = (np.arange(width) + 0.5) * array.shape[1] / width - 0.5 + coordinates = np.meshgrid(y, x, indexing="ij") + if array.ndim == 2: + return map_coordinates( + array, coordinates, order=1, mode="nearest", prefilter=False + ) + return np.stack( + [ + map_coordinates( + array[..., channel], + coordinates, + order=1, + mode="nearest", + prefilter=False, + ) + for channel in range(array.shape[-1]) + ], + axis=-1, + ) + + +def infer( + session, + image, + resolution_level, + *, + release_memory=True, + fov_x=None, + source_valid=None, +): + import numpy as np + + level = min(max(int(resolution_level), 0), 9) + tokens = int(MIN_TOKENS + level / 9.0 * (MAX_TOKENS - MIN_TOKENS)) + coord, encoder, normal, mask, scale = run_session( + session.backbone, + ["coord", "encoder", "normal", "mask", "scale"], + { + "image": np.ascontiguousarray( + image.transpose(2, 0, 1)[None], dtype=np.float32 + ), + "num_tokens": np.asarray(tokens, dtype=np.int64), + }, + release_memory=release_memory, + ) + for step in range(3): + if not np.isfinite(coord).all(): + raise ValueError("MoGe-3 produced non-finite coordinates") + inputs = build_sparse_inputs(coord) + inputs.update(coord=coord, encoder=encoder) + (delta,) = run_session( + session.refiner, + ["delta"], + inputs, + release_memory=release_memory and step == 2, + ) + coord[..., 2] += delta + del inputs, delta + height, width = image.shape[:2] + coord = _resize(coord[0], height, width) + depth = np.exp(coord[..., 2:]) + points = np.concatenate((coord[..., :2] * depth, depth), axis=-1) + normal = _resize(normal[0].transpose(1, 2, 0), height, width) + normal /= np.maximum(np.linalg.norm(normal, axis=-1, keepdims=True), 1e-12) + mask = 1.0 / (1.0 + np.exp(-np.clip(_resize(mask[0, 0], height, width), -80, 80))) + return postprocess( + { + "points": points[None], + "normal": normal[None], + "mask": mask[None], + "metric_scale": np.exp(scale), + }, + fov_x=fov_x, + source_valid=source_valid, + ) diff --git a/src/anyimage/server/models/upscale.py b/src/anyimage/server/models/upscale.py index 355dfc9..fdebe07 100644 --- a/src/anyimage/server/models/upscale.py +++ b/src/anyimage/server/models/upscale.py @@ -1,3 +1,4 @@ +from ..media.input import open_image from ..media.resolution import DEFAULT_MAX_AI_INPUT_SIZE, limit_image @@ -27,7 +28,7 @@ def infer_one( parameters["device"], ) try: - with Image.open(image_path) as opened: + with open_image(image_path) as opened: source = limit_image( opened, parameters.get("max_input_size", DEFAULT_MAX_AI_INPUT_SIZE), diff --git a/tests/anyimage/common/test_alpha_presentation.py b/tests/anyimage/common/test_alpha_presentation.py index f08c364..a5cb395 100644 --- a/tests/anyimage/common/test_alpha_presentation.py +++ b/tests/anyimage/common/test_alpha_presentation.py @@ -7,41 +7,39 @@ from PIL import Image from anyimage.common import image as images -from anyimage.operators.convert_to_plane.object import create_color_image -from anyimage.operators.cutout_tool.object import create_cropped_color_image +from tests.support.color_image import color_image @pytest.mark.parametrize("floating", [False, True]) def test_owned_color_images_preserve_source_and_use_premul(floating): original = bpy.data.images.new("Alpha source", width=2, height=2, alpha=True, float_buffer=floating) + if floating: + original.alpha_mode = "PREMUL" rgba = np.tile(np.array([0.2, 0.4, 0.8, 0.25], dtype=np.float32), (2, 2, 1)) original.pixels.foreach_set(rgba.ravel()) source = SimpleNamespace(data=original) expected = images.image_rgba(original) - copied = create_color_image(source) - cropped = create_cropped_color_image(source, (0, 0, 2, 2), expected) + cropped = color_image(source.data, (0, 0, 2, 2), expected) edited = images.create_image_edit_result(original, rgba.ravel(), (2, 2)) try: - assert original.alpha_mode == "STRAIGHT" + assert original.alpha_mode == ("PREMUL" if floating else "STRAIGHT") np.testing.assert_allclose(images.image_rgba(original), expected) assert edited.alpha_mode == original.alpha_mode - for image in (copied, cropped): - assert image.alpha_mode == "PREMUL" + for image in (cropped,): + assert image.alpha_mode == original.alpha_mode np.testing.assert_allclose(images.image_rgba(image), expected, atol=1 / 255) finally: - for image in (copied, cropped, edited, original): + for image in (cropped, edited, original): bpy.data.images.remove(image) -@pytest.mark.parametrize("loader", [images.load_image_edit_result, images.load_color_result_image]) -def test_job_loader_uses_edit_or_material_alpha_policy(tmp_path, loader): +def test_job_loader_preserves_edit_alpha_policy(tmp_path): rgba = np.array([[[51, 102, 204, 64]]], dtype=np.uint8) Image.fromarray(rgba).save(tmp_path / "image.0001.png") original = bpy.data.images.new("Job source", width=1, height=1, alpha=True) - source = original if loader == images.load_image_edit_result else SimpleNamespace(data=original) - result = loader(source, tmp_path) + result = images.load_image_edit_result(original, tmp_path) try: - assert result.alpha_mode == ("STRAIGHT" if loader == images.load_image_edit_result else "PREMUL") + assert result.alpha_mode == "STRAIGHT" assert result.packed_file is not None assert original.alpha_mode == "STRAIGHT" finally: diff --git a/tests/anyimage/common/test_color_image.py b/tests/anyimage/common/test_color_image.py new file mode 100644 index 0000000..00ed26c --- /dev/null +++ b/tests/anyimage/common/test_color_image.py @@ -0,0 +1,233 @@ +"""Verify isolated static color artifacts and source fidelity.""" + +from pathlib import Path +from types import SimpleNamespace + +import bpy +import numpy as np +import pytest +from PIL import Image + +from anyimage.common import color_image as colors, image as images, material + + +@pytest.fixture +def source(): + bpy.ops.wm.read_factory_settings(use_empty=True) + source = bpy.data.images.new("Source.png", width=4, height=2, alpha=True) + values = np.arange(32, dtype=np.float32).reshape(2, 4, 4) / 32 + values[0, 0, 3] = 0 + source.pixels.foreach_set(values.ravel()) + yield source + bpy.ops.wm.read_factory_settings(use_empty=True) + + +@pytest.mark.parametrize("floating,alpha", [(False, "STRAIGHT"), (False, "PREMUL"), (True, "STRAIGHT"), (True, "PREMUL"), (True, "CHANNEL_PACKED")]) +@pytest.mark.parametrize("bounds", [None, (1, 0, 3, 2)]) +def test_color_artifact_preserves_bounds_and_library_reload(source, tmp_path, monkeypatch, floating, alpha, bounds): + if floating: + source = bpy.data.images.new("Source.exr", width=4, height=2, alpha=True, float_buffer=True) + source.pixels.foreach_set(np.tile(np.array([4, 2, .5, .25, 8, 3, 1, 0], dtype=np.float32), 4)) + source.alpha_mode = alpha + source["anyimage_clipboard_sha256"] = "original" + before = images.image_rgba(source) + state = (source.filepath_raw, source.colorspace_settings.name, source.alpha_mode) + path = colors.prepare_material_color_input(source, bounds) + color = colors.load_material_color_image(path) + try: + assert color != source and color.name.endswith("_color" + path.suffix) + assert "anyimage_clipboard_sha256" not in color + assert Path(color.filepath_raw) == path + expected = before if bounds is None else before[bounds[1]:bounds[3], bounds[0]:bounds[2]] + np.testing.assert_allclose(images.image_rgba(color), expected, atol=1e-6 if floating else 1/255) + monkeypatch.setattr(material, "configured_material_view_adaptation", lambda: True) + result = material.create_image_material(source, color) + assert next(n.image for n in result.node_tree.nodes if n.type == "TEX_IMAGE") == color + colors.cleanup_material_color_input(path) + assert not path.parent.exists() + library = tmp_path / "colors.blend" + bpy.data.libraries.write(str(library), {color}) + with bpy.data.libraries.load(str(library)) as (saved, loaded): + loaded.images = saved.images + np.testing.assert_allclose(images.image_rgba(loaded.images[0]), expected, atol=1e-6 if floating else 1/255) + if floating: + assert color.colorspace_settings.name == "Linear Rec.709" + assert color.alpha_mode == ("PREMUL" if alpha == "STRAIGHT" else alpha) + layer = next(n for n in result.node_tree.nodes if n.type == "GROUP") + assert layer.inputs["Alpha Fix"].default_value == 0 + assert (source.filepath_raw, source.colorspace_settings.name, source.alpha_mode) == state + np.testing.assert_array_equal(images.image_rgba(source), before) + finally: + colors.cleanup_material_color_input(path) + + +def test_original_path_never_matches_material_color(tmp_path, monkeypatch): + path = tmp_path / "original.png" + Image.new("RGBA", (2, 2), (100, 150, 200, 128)).save(path) + source = bpy.data.images.load(str(path), check_existing=False) + with colors.material_color_image(source) as color: + result = material.create_image_material(source, color) + color.pixels[0] = 1 + bpy.data.images.remove(source) + imported = bpy.data.images.load(str(path), check_existing=True) + assert imported != color + assert imported.alpha_mode == "STRAIGHT" + assert images.image_pixels(imported)[0] == pytest.approx(100/255, abs=1/255) + bpy.data.materials.remove(result) + bpy.data.images.remove(imported) + + +@pytest.mark.parametrize("kind", ["MOVIE", "SEQUENCE"]) +def test_static_input_rejects_single_frame_animation(kind): + with pytest.raises(ValueError, match="static images"): + colors.require_static_color_image(SimpleNamespace(source=kind, frame_duration=1)) + + +@pytest.mark.parametrize("stage", ["export", "load", "material", "object"]) +def test_failed_color_preparation_releases_owned_resources(source, tmp_path, monkeypatch, stage): + directory = tmp_path / "owned-color" + directory.mkdir() + monkeypatch.setattr(colors.tempfile, "mkdtemp", lambda **_: str(directory)) + before = set(bpy.data.images) + original = images.image_pixels(source) + def fail(*_, **__): + raise RuntimeError("Color failure") + if stage == "export": + monkeypatch.setattr(colors, "_save_byte_color", fail) + elif stage == "load": + monkeypatch.setattr(colors, "load_material_color_image", fail) + elif stage == "material": + monkeypatch.setattr(material, "material_node_group", fail) + result = None + try: + with pytest.raises(RuntimeError, match="Color failure"): + with colors.material_color_image(source) as color: + result = material.create_image_material(source, color) + fail() + finally: + if result is not None: + bpy.data.materials.remove(result) + assert set(bpy.data.images) == before + assert not directory.exists() + np.testing.assert_array_equal(images.image_pixels(source), original) + + +def test_ai_color_uses_submitted_crop_after_source_changes(source): + from anyimage.operators.cutout_tool.operators import _generated_color_result + bounds = (1, 0, 3, 2) + expected = images.image_rgba(source)[:, 1:3].copy() + with colors.material_color_image(source, bounds=bounds) as direct: + path = colors.prepare_material_color_input(source, bounds) + try: + assert colors.material_analysis_input(path) == path + source.pixels.foreach_set(np.zeros(32, np.float32)) + result, alpha, result_bounds = _generated_color_result(path, bounds) + try: + assert result_bounds == bounds and tuple(result.size) == (2, 2) + np.testing.assert_array_equal(images.image_rgba(result), images.image_rgba(direct)) + np.testing.assert_allclose(images.image_rgba(result), expected, atol=1/255) + np.testing.assert_allclose(alpha, expected[..., 3], atol=1/255) + finally: + bpy.data.images.remove(result) + finally: + colors.cleanup_material_color_input(path) + + +@pytest.mark.parametrize("color_space", ["Linear Rec.709", "sRGB", "Non-Color"]) +def test_float_straight_uses_exr_interpretation_without_rejecting_hidden_rgb(source, tmp_path, color_space): + source = bpy.data.images.new("Straight HDR", width=2, height=1, alpha=True, float_buffer=True) + source.colorspace_settings.name = color_space + rgba = np.array([4, 2, .5, .25, 8, 3, 1, 0], np.float32) + source.pixels.foreach_set(rgba) + with colors.material_color_image(source) as color: + assert color.alpha_mode == "PREMUL" + assert color.colorspace_settings.name == "Linear Rec.709" + library = tmp_path / "straight.blend" + bpy.data.libraries.write(str(library), {color}) + with bpy.data.libraries.load(str(library)) as (saved, loaded): + loaded.images = saved.images + np.testing.assert_array_equal(loaded.images[0].pixels[:], rgba) + assert (source.colorspace_settings.name, source.alpha_mode) == (color_space, "STRAIGHT") + np.testing.assert_array_equal(source.pixels[:], rgba) + + +def test_hdr_analysis_preview_is_separate_from_full_precision_color(source): + source = bpy.data.images.new("HDR", width=2, height=1, alpha=True, float_buffer=True) + source.alpha_mode = "PREMUL" + source.pixels.foreach_set(np.array([4, 2, .5, .25, 8, 3, 1, 0], np.float32)) + path = colors.prepare_material_color_input(source) + try: + preview = colors.material_analysis_input(path) + assert preview != path and preview.suffix == ".png" and path.suffix == ".exr" + with Image.open(preview) as image: + assert image.size == tuple(source.size) + color = colors.load_material_color_image(path) + try: + np.testing.assert_array_equal(images.image_rgba(color), images.image_rgba(source)) + finally: + bpy.data.images.remove(color) + finally: + colors.cleanup_material_color_input(path) + assert not preview.exists() + + +@pytest.mark.parametrize("alpha_mode", ["STRAIGHT", "PREMUL", "CHANNEL_PACKED", "NONE"]) +@pytest.mark.parametrize("source_kind", ["generated", "file", "dirty", "packed"]) +def test_float_color_conversion_preserves_rendered_color_and_opacity(source, tmp_path, alpha_mode, source_kind): + from anyimage.common.hdr_image import save_hdr_result + + rgba = np.array([[[8, 3, 1, 0], [4, 2, .5, .25], [2, 1, .25, .5], [1, .5, .125, 1]]], np.float32) + if source_kind != "generated": + path = tmp_path / "source.exr" + save_hdr_result(rgba, path) + source = bpy.data.images.load(str(path), check_existing=False) + else: + source = bpy.data.images.new("Float Render", width=4, height=1, alpha=True, float_buffer=True) + source.pixels.foreach_set(rgba.ravel()) + source.alpha_mode = alpha_mode + if source_kind == "dirty": + source.pixels.foreach_set(rgba.ravel()) + elif source_kind == "packed": + source.pack() + scene = bpy.context.scene + scene.render.engine = "CYCLES" + scene.cycles.device = "CPU" + scene.cycles.samples = 16 + scene.cycles.use_denoising = False + scene.render.film_transparent = True + scene.render.resolution_x, scene.render.resolution_y = 64, 16 + scene.render.resolution_percentage = 100 + scene.render.image_settings.file_format = "OPEN_EXR" + scene.render.image_settings.color_mode = "RGBA" + scene.render.image_settings.color_depth = "32" + bpy.ops.object.camera_add(location=(0, 0, 5)) + scene.camera = bpy.context.object + scene.camera.data.type = "ORTHO" + scene.camera.data.ortho_scale = 4 + bpy.ops.mesh.primitive_plane_add(size=2) + bpy.context.object.scale = (2, .5, 1) + result = bpy.data.materials.new("Float Interpretation") + result.use_nodes = True + nodes, links = result.node_tree.nodes, result.node_tree.links + nodes.clear() + texture = nodes.new("ShaderNodeTexImage") + texture.interpolation = "Closest" + emission = nodes.new("ShaderNodeEmission") + transparent = nodes.new("ShaderNodeBsdfTransparent") + mix = nodes.new("ShaderNodeMixShader") + output = nodes.new("ShaderNodeOutputMaterial") + links.new(texture.outputs["Color"], emission.inputs["Color"]) + links.new(texture.outputs["Alpha"], mix.inputs[0]) + links.new(transparent.outputs[0], mix.inputs[1]) + links.new(emission.outputs[0], mix.inputs[2]) + links.new(mix.outputs[0], output.inputs["Surface"]) + bpy.context.object.data.materials.append(result) + with colors.material_color_image(source) as color: + rendered = [] + for name, image in (("before", source), ("after", color)): + texture.image = image + scene.render.filepath = str(tmp_path / (name + ".exr")) + bpy.ops.render.render(write_still=True) + image = bpy.data.images.load(scene.render.filepath, check_existing=False) + rendered.append(images.image_rgba(image)) + np.testing.assert_allclose(rendered[1], rendered[0], rtol=2e-5, atol=1e-6) diff --git a/tests/anyimage/common/test_hdr_image.py b/tests/anyimage/common/test_hdr_image.py new file mode 100644 index 0000000..43f1938 --- /dev/null +++ b/tests/anyimage/common/test_hdr_image.py @@ -0,0 +1,189 @@ +from types import SimpleNamespace +from unittest.mock import patch + +import bpy +import numpy as np +import pytest +from PIL import Image + +from anyimage.common.hdr_image import HdrBackgroundInput, recognition_rgba, save_hdr_result +from anyimage.common.image import cleanup_image_input, image_content_state, image_rgba, restore_image_content +from anyimage.common.image_target import ImageEditTarget +from anyimage.common import image_target as targets +from tests.support.image_texture import texture, texture_context +from tests.support.hdr_image import hdr_texture + + +@pytest.mark.parametrize("source_kind", ["generated", "file", "packed", "dirty"]) +@pytest.mark.parametrize("owner_kind", ["texture", "empty"]) +@pytest.mark.parametrize("shared", [False, True]) +def test_hdr_commit_keeps_float_pixels_and_transaction(hdr_texture, tmp_path, source_kind, owner_kind, shared): + texture = hdr_texture + source = texture.image + rgba = image_rgba(source).copy() + if source_kind != "generated": + path = tmp_path / "source.exr" + save_hdr_result(rgba, path) + source = bpy.data.images.load(str(path), check_existing=False) + texture.image = source + if source_kind == "packed": + source.pack() + if source_kind == "dirty": + rgba[1, 1, 0] = 19 + source.pixels.foreach_set(np.flipud(rgba).ravel()) + source.use_fake_user = True + if owner_kind == "empty": + owner = bpy.data.objects.new("HDR Empty", None) + owner.empty_display_type = "IMAGE" + owner.data = source + bpy.context.scene.collection.objects.link(owner) + texture.image = None + context = SimpleNamespace(object=owner, space_data=None) + else: + owner = texture + context = texture_context(texture) + if shared: + other = texture.id_data.nodes.new("ShaderNodeTexImage") + other.image = source + target = ImageEditTarget.capture(context) + snapshot = HdrBackgroundInput.prepare(source) + original_state = image_content_state(source) + alpha = np.linspace(0.03, 0.91, 12, dtype=np.float32).reshape(3, 4) + alpha_path = tmp_path / "alpha.npy" + np.save(alpha_path, alpha) + try: + with Image.open(snapshot.path) as preview: + assert preview.size == (4, 3) + assert np.asarray(preview)[0, 0, :3].max() == 0 + result = snapshot.apply(target, alpha_path) + expected = rgba.copy() + expected[..., 3] *= alpha + assert result.is_float and result.packed_file and result.alpha_mode == "CHANNEL_PACKED" + np.testing.assert_allclose(image_rgba(result), expected, atol=1e-7) + assert (result == source) == (not shared) + if shared: + np.testing.assert_array_equal(image_rgba(other.image), rgba) + else: + assert result.use_fake_user + # The encoded result restores the same ID and floats after a source-state restore. + result_state = image_content_state(result) + restore_image_content(result, original_state) + np.testing.assert_allclose(image_rgba(result), rgba, atol=1e-7) + restore_image_content(result, result_state) + np.testing.assert_allclose(image_rgba(result), expected, atol=1e-7) + blend_path = tmp_path / "roundtrip.blend" + result_name = result.name + bpy.ops.wm.save_as_mainfile(filepath=str(blend_path)) + bpy.ops.wm.open_mainfile(filepath=str(blend_path)) + restored = bpy.data.images[result_name] + assert restored.is_float and restored.packed_file + np.testing.assert_allclose(image_rgba(restored), expected, atol=1e-7) + finally: + cleanup_image_input(snapshot.path, True) + + +@pytest.mark.parametrize("invalid", ["shape", "dtype", "nan", "range", "corrupt", "source", "colorspace", "binding"]) +def test_hdr_rejects_invalid_result_and_source_changes(hdr_texture, tmp_path, invalid): + target = ImageEditTarget.capture(texture_context(hdr_texture)) + snapshot = HdrBackgroundInput.prepare(target.image) + alpha = np.ones((3, 4), np.float32) + if invalid == "shape": alpha = alpha[:1] + if invalid == "dtype": alpha = alpha.astype(np.float64) + if invalid == "nan": alpha[0, 0] = np.nan + if invalid == "range": alpha[0, 0] = 2 + path = tmp_path / "alpha.npy" + np.save(path, alpha) + if invalid == "corrupt": path.write_bytes(b"broken") + if invalid == "source": target.image.pixels[4] = 20 + if invalid == "colorspace": target.image.colorspace_settings.name = "ACEScg" + if invalid == "binding": hdr_texture.image = bpy.data.images.new("Other", width=1, height=1) + before = image_rgba(hdr_texture.image).copy() + count = len(bpy.data.images) + try: + with pytest.raises(RuntimeError): + snapshot.apply(target, path) + np.testing.assert_array_equal(image_rgba(hdr_texture.image), before) + assert len(bpy.data.images) == count + finally: + cleanup_image_input(snapshot.path, True) + + +def test_hdr_preview_is_fixed_and_preserves_source(hdr_texture): + image = hdr_texture.image + before = image_rgba(image).copy() + first = HdrBackgroundInput.prepare(image) + bpy.context.scene.view_settings.view_transform = "Standard" + bpy.context.scene.view_settings.exposure = 7 + second = HdrBackgroundInput.prepare(image) + try: + with Image.open(first.path) as a, Image.open(second.path) as b: + np.testing.assert_array_equal(np.asarray(a), np.asarray(b)) + np.testing.assert_array_equal(image_rgba(image), before) + np.testing.assert_array_equal(recognition_rgba(np.zeros((1, 1, 4), np.float32)), [[[0, 0, 0, 1]]]) + finally: + cleanup_image_input(first.path, True) + cleanup_image_input(second.path, True) + + +def test_hdr_rejects_animation_before_export(hdr_texture): + hdr_texture.image.source = "SEQUENCE" + with pytest.raises(RuntimeError, match="still images"): + HdrBackgroundInput.prepare(hdr_texture.image) + + +def test_hdr_export_failure_releases_blender_resources(hdr_texture, tmp_path): + images, scenes = len(bpy.data.images), len(bpy.data.scenes) + blocked = tmp_path / "file" + blocked.write_bytes(b"not a directory") + with pytest.raises(RuntimeError): + save_hdr_result(image_rgba(hdr_texture.image), blocked / "output.exr") + assert (len(bpy.data.images), len(bpy.data.scenes)) == (images, scenes) + + +def test_straight_exr_only_changes_alpha(hdr_texture, tmp_path): + path = tmp_path / "straight.exr" + encoded = image_rgba(hdr_texture.image).copy() + save_hdr_result(encoded, path) + source = bpy.data.images.load(str(path), check_existing=False) + source.alpha_mode = "STRAIGHT" + hdr_texture.image = source + before = image_rgba(source).copy() + np.testing.assert_allclose(before[..., :3], encoded[..., :3] * encoded[..., 3:4], atol=1e-7) + snapshot = HdrBackgroundInput.prepare(source) + alpha_path = tmp_path / "alpha.npy" + np.save(alpha_path, np.full((3, 4), 0.3, np.float32)) + try: + result = snapshot.apply(ImageEditTarget.capture(texture_context(hdr_texture)), alpha_path) + rgba = image_rgba(result) + np.testing.assert_array_equal(rgba[..., :3], before[..., :3]) + np.testing.assert_allclose(rgba[..., 3], before[..., 3] * 0.3, atol=1e-7) + finally: + cleanup_image_input(snapshot.path, True) + + +def test_hdr_failed_commit_restores_unsaved_source(hdr_texture, tmp_path): + target = ImageEditTarget.capture(texture_context(hdr_texture)) + snapshot = HdrBackgroundInput.prepare(target.image) + before = image_rgba(target.image).copy() + original_source = target.image.source + alpha_path = tmp_path / "alpha.npy" + np.save(alpha_path, np.full((3, 4), 0.3, np.float32)) + calls = 0 + + def fail_first(image, state): + nonlocal calls + calls += 1 + restore_image_content(image, state) + if calls == 1: + raise RuntimeError("Commit failed") + + count = len(bpy.data.images) + try: + with patch.object(targets, "restore_image_content", side_effect=fail_first): + with pytest.raises(RuntimeError, match="Commit failed"): + snapshot.apply(target, alpha_path, prepare_undo=True) + np.testing.assert_array_equal(image_rgba(target.image), before) + assert target.image.source == original_source + assert len(bpy.data.images) == count + finally: + cleanup_image_input(snapshot.path, True) diff --git a/tests/anyimage/common/test_hdr_rgb_render.py b/tests/anyimage/common/test_hdr_rgb_render.py new file mode 100644 index 0000000..b6e27e4 --- /dev/null +++ b/tests/anyimage/common/test_hdr_rgb_render.py @@ -0,0 +1,80 @@ +import bpy +import numpy as np +import pytest + +from anyimage.common.hdr_image import HdrBackgroundInput +from anyimage.common.image import cleanup_image_input, image_rgba +from anyimage.common.image_target import ImageEditTarget +from tests.support.hdr_image import hdr_texture +from tests.support.image_texture import texture, texture_context + + +@pytest.mark.parametrize("use_alpha", [False, True]) +def test_hdr_texture_rgb_remains_visible_independently_of_alpha(hdr_texture, tmp_path, use_alpha): + source = hdr_texture.image + source.pixels.foreach_set(np.tile([4, 2, 0.5, 1], 12).astype(np.float32)) + alpha = np.tile([0, 0.25, 0.5, 1], (3, 1)).astype(np.float32) + path = tmp_path / "alpha.npy" + np.save(path, alpha) + snapshot = HdrBackgroundInput.prepare(source) + try: + result = snapshot.apply(ImageEditTarget.capture(texture_context(hdr_texture)), path) + finally: + cleanup_image_input(snapshot.path, True) + + # A second removal must accept its own independent-channel result and keep RGB intact. + snapshot = HdrBackgroundInput.prepare(result) + try: + snapshot.apply(ImageEditTarget.capture(texture_context(hdr_texture)), path) + np.testing.assert_array_equal(image_rgba(result)[..., :3], np.tile([4, 2, 0.5], (3, 4, 1))) + finally: + cleanup_image_input(snapshot.path, True) + result.reload() + + scene = bpy.context.scene + scene.render.engine = "CYCLES" + scene.cycles.device = "CPU" + scene.cycles.samples = 32 + scene.cycles.use_denoising = False + scene.render.film_transparent = True + scene.render.resolution_x, scene.render.resolution_y = 64, 16 + scene.render.resolution_percentage = 100 + scene.render.image_settings.file_format = "OPEN_EXR" + scene.render.image_settings.color_mode = "RGBA" + scene.render.image_settings.color_depth = "32" + scene.render.filepath = str(tmp_path / "render.exr") + bpy.ops.object.camera_add(location=(0, 0, 5)) + scene.camera = bpy.context.object + scene.camera.data.type = "ORTHO" + scene.camera.data.ortho_scale = 4 + bpy.ops.mesh.primitive_plane_add(size=2) + bpy.context.object.scale = (2, 0.5, 1) + material = bpy.data.materials.new("HDR RGB validation") + material.use_nodes = True + nodes, links = material.node_tree.nodes, material.node_tree.links + nodes.clear() + image_node = nodes.new("ShaderNodeTexImage") + image_node.image = result + image_node.interpolation = "Closest" + emission = nodes.new("ShaderNodeEmission") + links.new(image_node.outputs["Color"], emission.inputs["Color"]) + output = nodes.new("ShaderNodeOutputMaterial") + if use_alpha: + transparent = nodes.new("ShaderNodeBsdfTransparent") + mix = nodes.new("ShaderNodeMixShader") + links.new(image_node.outputs["Alpha"], mix.inputs[0]) + links.new(transparent.outputs[0], mix.inputs[1]) + links.new(emission.outputs[0], mix.inputs[2]) + links.new(mix.outputs[0], output.inputs["Surface"]) + else: + links.new(emission.outputs[0], output.inputs["Surface"]) + bpy.context.object.data.materials.append(material) + bpy.ops.render.render(write_still=True) + rendered = bpy.data.images.load(scene.render.filepath, check_existing=False) + pixels = image_rgba(rendered) + for index, opacity in enumerate((0, 0.0625, 0.25, 1)): + patch = pixels[4:12, index * 16 + 4:index * 16 + 12] + expected_alpha = opacity if use_alpha else 1 + np.testing.assert_allclose(patch[..., 3].mean(), expected_alpha, atol=0.035) + expected_rgb = patch[..., 3:4] * np.array([4, 2, 0.5]) + np.testing.assert_allclose(patch[..., :3], expected_rgb, rtol=2e-5, atol=1e-6) diff --git a/tests/anyimage/common/test_image_data.py b/tests/anyimage/common/test_image_data.py index 8d00e55..f28d98f 100644 --- a/tests/anyimage/common/test_image_data.py +++ b/tests/anyimage/common/test_image_data.py @@ -217,13 +217,6 @@ class ImageDataTest(unittest.TestCase): ) - def test_color_name_strips_video_extension(self): - source_object = SimpleNamespace(data=SimpleNamespace(name="clip.mp4")) - - self.assertEqual( - self.modules.image.color_data_name(source_object), - "clip_color.png", - ) def test_force_sequence_source_keeps_loaded_sequence(self): @@ -322,20 +315,6 @@ class ImageDataTest(unittest.TestCase): ) - def test_color_name_uses_source_image_base_name(self): - source_object = SimpleNamespace( - data=SimpleNamespace(name="img.png"), - ) - - self.assertEqual( - self.modules.image.color_data_name(source_object), - "img_color.png", - ) - source_object.data.name = "img.png.001" - self.assertEqual( - self.modules.image.color_data_name(source_object), - "img.001_color.png", - ) def test_image_empty_bounds_keep_image_aspect_and_empty_offset(self): diff --git a/tests/anyimage/common/test_image_materials.py b/tests/anyimage/common/test_image_materials.py index b1171b2..a633d25 100644 --- a/tests/anyimage/common/test_image_materials.py +++ b/tests/anyimage/common/test_image_materials.py @@ -183,7 +183,7 @@ def test_loaded_normal_image_uses_normal_texture_name(tmp_path): assert image.packed_file is not None -def test_converted_color_images_are_new_and_use_blender_duplicate_suffix(): +def test_shared_conversion_images_use_blender_duplicate_suffix(): bpy.ops.wm.read_factory_settings(use_empty=True) image = bpy.data.images.new("Poster", width=16, height=16) source = bpy.data.objects.new("Poster", None) @@ -191,8 +191,11 @@ def test_converted_color_images_are_new_and_use_blender_duplicate_suffix(): source.data = image bpy.context.collection.objects.link(source) - first = plane_object.create_color_image(source) - second = plane_object.create_color_image(source) + other = source.copy() + bpy.context.collection.objects.link(other) + from tests.support.color_image import color_image + first = color_image(source.data) + second = color_image(source.data) assert first is not image assert second is not image diff --git a/tests/anyimage/common/test_image_target.py b/tests/anyimage/common/test_image_target.py index 4927431..2b689f6 100644 --- a/tests/anyimage/common/test_image_target.py +++ b/tests/anyimage/common/test_image_target.py @@ -80,7 +80,7 @@ def test_invalid_node_context_does_not_fall_back_to_empty(texture, invalid): targets.ImageEditTarget.capture(context) -@pytest.mark.parametrize("change", ["delete", "recreate", "rebind", "delete_material", "delete_image"]) +@pytest.mark.parametrize("change", ["delete", "recreate", "rebind", "delete_material", "delete_image", "identity"]) def test_target_rejects_changed_identity_before_loading_result(texture, tmp_path, change): target = targets.ImageEditTarget.capture(texture_context(texture)) tree = texture.id_data @@ -95,6 +95,9 @@ def test_target_rejects_changed_identity_before_loading_result(texture, tmp_path texture.image = bpy.data.images.new("Replacement", width=2, height=2) elif change == "delete_material": bpy.data.materials.remove(bpy.data.materials["Texture material"]) + elif change == "identity": + # A reused RNA address must not make a replacement node the original target. + texture["anyimage_identity"] = "replacement" else: bpy.data.images.remove(texture.image) count = len(bpy.data.images) diff --git a/tests/anyimage/common/test_input_preparation.py b/tests/anyimage/common/test_input_preparation.py new file mode 100644 index 0000000..ff1180c --- /dev/null +++ b/tests/anyimage/common/test_input_preparation.py @@ -0,0 +1,54 @@ +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from tests.support.blender import BlenderTestCase + + +class InputPreparationTest(BlenderTestCase): + def test_sequence_format_is_checked_before_copy(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "frame.0001.exr" + path.touch() + image = SimpleNamespace(source="SEQUENCE") + with ( + patch.object(self.image_data, "image_path", return_value=path), + patch.object(self.image_data, "image_is_packed", return_value=False), + patch.object(self.image_data.tempfile, "mkdtemp") as create, + ): + with self.assertRaisesRegex(RuntimeError, "Unsupported input format: .exr"): + self.image_data.prepare_image_input(image, None) + create.assert_not_called() + self.assertTrue(path.exists()) + + def test_packed_export_failure_removes_temporary_directory(self): + for failure in (OSError("disk full"), None): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "export" + output.mkdir() + image = SimpleNamespace(file_format="OPEN_EXR", save=Mock(side_effect=failure)) + with patch.object(self.image_data.tempfile, "mkdtemp", return_value=str(output)): + with self.assertRaises(RuntimeError): + self.image_data.save_packed_image(image, None) + self.assertEqual(image.file_format, "OPEN_EXR") + self.assertFalse(output.exists()) + + def test_sequence_copy_failure_preserves_originals(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "frame.0001.png" + source.write_bytes(b"source") + output = root / "prepared" + output.mkdir() + with ( + patch.object(self.image_data, "image_path", return_value=source), + patch.object(self.image_data, "image_is_packed", return_value=False), + patch.object(self.image_data, "collect_sequence_frames", return_value=[source]), + patch.object(self.image_data.tempfile, "mkdtemp", return_value=str(output)), + patch.object(self.image_data.shutil, "copy2", side_effect=OSError("disk full")), + ): + with self.assertRaises(OSError): + self.image_data.prepare_image_input(SimpleNamespace(source="SEQUENCE"), None) + self.assertFalse(output.exists()) + self.assertEqual(source.read_bytes(), b"source") diff --git a/tests/anyimage/common/test_material_alpha.py b/tests/anyimage/common/test_material_alpha.py index 8a8c7cb..d8532cb 100644 --- a/tests/anyimage/common/test_material_alpha.py +++ b/tests/anyimage/common/test_material_alpha.py @@ -14,11 +14,19 @@ def test_material_alpha_strength_follows_preference(image_layer, monkeypatch, en color = bpy.data.images.new("Alpha edge", width=2, height=2, alpha=True) scene = SimpleNamespace(view_settings=SimpleNamespace(view_transform=view)) result = material.create_image_material(color, color, shadeless=shadeless, scene=scene) + if shadeless: + shader = next(node for node in result.node_tree.nodes if node.type == "GROUP") + assert shader.node_tree.name == "O Shadeless" + assert shader.inputs["Color"].links[0].from_node.type == "TEX_IMAGE" + assert shader.inputs["Alpha"].links[0].from_socket.name == "Alpha" + assert not any(node.type in {"MIX_SHADER", "BSDF_TRANSPARENT"} for node in result.node_tree.nodes) + assert len([node for node in result.node_tree.nodes if node.type == "GROUP"]) == 1 + return layer = next(node for node in result.node_tree.nodes if node.type == "GROUP" and node.node_tree == image_layer) assert layer.inputs["Alpha Fix"].default_value == float(enabled) alpha_link = layer.outputs["Alpha"].links[0] - assert alpha_link.to_node.type == ("MIX_SHADER" if shadeless else "BSDF_PRINCIPLED") - assert alpha_link.to_socket.name == ("Fac" if shadeless else "Alpha") + assert alpha_link.to_node.type == "BSDF_PRINCIPLED" + assert alpha_link.to_socket.name == "Alpha" layer.inputs["Alpha Fix"].default_value = 0.5 monkeypatch.setattr(material, "configured_material_view_adaptation", lambda: not enabled) second = material.create_image_material(color, color, shadeless=shadeless, scene=scene) diff --git a/tests/anyimage/common/test_material_view_adaptation.py b/tests/anyimage/common/test_material_view_adaptation.py index de54e1c..210b0e0 100644 --- a/tests/anyimage/common/test_material_view_adaptation.py +++ b/tests/anyimage/common/test_material_view_adaptation.py @@ -10,8 +10,7 @@ from anyimage import preferences from anyimage.common import image as images from anyimage.common import material from anyimage.operators.clipboard_image import actions as clipboard -from anyimage.operators.convert_to_plane.object import create_color_image -from anyimage.operators.cutout_tool.object import create_cropped_color_image +from tests.support.color_image import color_image @pytest.fixture @@ -74,10 +73,7 @@ def test_color_space_and_alpha_follow_preference(color_data, enabled, view, spac @pytest.mark.parametrize("enabled", [False, True]) @pytest.mark.parametrize("kind", [ "byte", "packed", "file", "dirty", - pytest.param("float", marks=pytest.mark.xfail( - strict=True, - reason="Native float RGB is reinterpreted by Blender after packing and reloading with a display color space", - )), + "float", ]) def test_material_pixels_survive_pack_and_library_reload(color_data, tmp_path, enabled, kind): expected = np.array([[[51, 102, 204, 64], [153, 76, 25, 0]]], dtype=np.uint8) @@ -100,8 +96,7 @@ def test_material_pixels_survive_pack_and_library_reload(color_data, tmp_path, e owner.data = source with patch.object(material, "configured_material_view_adaptation", return_value=enabled): result = material.configure_material_color_image(source, view_scene("AgX")) - assert result != source - assert source.alpha_mode == "PREMUL" + assert result == source np.testing.assert_allclose(images.image_pixels(source), before, atol=1 / 255) np.testing.assert_allclose(images.image_pixels(result), before, atol=1 / 255) if result.packed_file is None or result.is_dirty: @@ -121,7 +116,7 @@ def test_material_pixels_survive_pack_and_library_reload(color_data, tmp_path, e @pytest.mark.parametrize("shadeless", [False, True]) @pytest.mark.parametrize("entry", ["plane", "cutout", "depth", "clipboard"]) -def test_new_material_uses_current_preference_and_preserves_existing(color_data, entry, shadeless): +def test_material_configures_the_image_selected_by_its_caller(color_data, entry, shadeless): source = bpy.data.images.new("Shared source", width=2, height=2, alpha=True) source.pixels.foreach_set(np.tile(np.array([0.2, 0.4, 0.8, 0.25], dtype=np.float32), 4)) owner = bpy.data.objects.new("Shared Empty", None) @@ -144,10 +139,8 @@ def test_new_material_uses_current_preference_and_preserves_existing(color_data, if entry == "clipboard": first = clipboard.create_image_material(source, source, shadeless=shadeless, scene=scene) else: - if entry == "plane": - color = create_color_image(owner) - elif entry == "cutout": - color = create_cropped_color_image(owner, (0, 0, 2, 2), images.image_rgba(source)) + if entry == "cutout": + color = color_image(owner.data, (0, 0, 2, 2), images.image_rgba(source)) else: color = source first = material.create_image_material(owner.data, color, 0, shadeless=shadeless, scene=scene, **data_textures) @@ -161,15 +154,15 @@ def test_new_material_uses_current_preference_and_preserves_existing(color_data, else: second = material.create_image_material(owner.data, original_color, 0, shadeless=shadeless, scene=scene) new_color = color_texture(second) - assert new_color != original_color + assert new_color == original_color assert new_color.colorspace_settings.name == "sRGB" assert new_color.alpha_mode == "PREMUL" - assert original_color.colorspace_settings.name == "AgX Base sRGB" - assert original_color.alpha_mode == "STRAIGHT" + assert original_color.colorspace_settings.name == "sRGB" + assert original_color.alpha_mode == "PREMUL" assert not settings.adapt_material_to_view_transform assert owner.data == source assert source.colorspace_settings.name == "sRGB" - assert source.alpha_mode == "STRAIGHT" + assert source.alpha_mode == ("STRAIGHT" if entry == "cutout" else "PREMUL") for image in (source, original_color, new_color): np.testing.assert_allclose(images.image_pixels(image), before, atol=1 / 255) for key, image in data_textures.items(): diff --git a/tests/anyimage/operators/test_ai_image_operators.py b/tests/anyimage/operators/test_ai_image_operators.py index 5885ac8..c0a938c 100644 --- a/tests/anyimage/operators/test_ai_image_operators.py +++ b/tests/anyimage/operators/test_ai_image_operators.py @@ -53,14 +53,15 @@ class AiImageOperatorsTest(BlenderTestCase): def test_upscale_operator_opens_setup_when_required_ai_is_missing(self): expected = {"CANCELLED"} + operator = self.upscale.UpscaleImage() with patch.object( self.upscale, "invoke_ai_setup_if_needed", return_value=expected, ) as setup: - result = self.upscale.UpscaleImage().invoke(None, None) + result = operator.invoke(None, None) - setup.assert_called_once_with() + setup.assert_called_once_with(operator) self.assertIs(result, expected) diff --git a/tests/anyimage/operators/test_ai_input_failures.py b/tests/anyimage/operators/test_ai_input_failures.py new file mode 100644 index 0000000..6d006d4 --- /dev/null +++ b/tests/anyimage/operators/test_ai_input_failures.py @@ -0,0 +1,96 @@ +import tempfile +from contextlib import ExitStack +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from tests.support.blender import BlenderTestCase + + +class AiInputFailuresTest(BlenderTestCase): + def test_image_edits_reject_same_input_before_submission(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input.tiff" + path.touch() + messages = [] + for module, operator_type in ( + (self.upscale, self.upscale.UpscaleImage), + (self.remove_background, self.remove_background.RunBackgroundRemoval), + ): + operator = operator_type() + operator.report = Mock() + target = Mock(image=SimpleNamespace(size=(16, 16)), start_frame=1) + target.prepare.return_value = (path, False) + context = SimpleNamespace(scene=SimpleNamespace( + anyimage_settings=SimpleNamespace(video_frames=1))) + with ExitStack() as stack: + stack.enter_context(patch.object(module.ImageEditTarget, "capture", return_value=target)) + stack.enter_context(patch.object(module, "require_environment")) + stack.enter_context(patch.object(module, "require_model")) + stack.enter_context(patch.object(self.runtime.runtime, "active_job", None)) + submit = stack.enter_context(patch.object(self.runtime.runtime, "begin_job")) + if module == self.upscale: + stack.enter_context(patch.object(module, "configured_upscale_model", return_value="SAFMN_REAL_X4")) + stack.enter_context(patch.object(module, "configured_max_ai_input_size", return_value=2048)) + self.assertEqual(operator.execute(context), {"CANCELLED"}) + submit.assert_not_called() + messages.append(operator.report.call_args.args[1]) + self.assertIsNone(operator._image_target) + self.assertTrue(path.exists()) + self.assertEqual(messages[0], messages[1]) + self.assertIn("Unsupported input format: .tiff", messages[0]) + + def test_remove_background_catches_nested_error(self): + operator = self.remove_background.RemoveImageBackground() + operator.report = Mock() + ops = SimpleNamespace(anyimage=SimpleNamespace( + run_background_removal=Mock(side_effect=RuntimeError("Error: invalid input\n")))) + with patch.object(self.remove_background.bpy, "ops", ops, create=True): + self.assertEqual(operator.execute(None), {"CANCELLED"}) + operator.report.assert_called_once_with({"ERROR"}, "invalid input") + + def test_setup_failures_cancel_at_each_boundary(self): + for ready in (True, False): + operator = self.ai_setup.SetupAIEnvironment() + operator.use_mirror = False + operator.report = Mock() + failing = Mock(side_effect=RuntimeError("Error: setup failed\n")) + ops = SimpleNamespace(anyimage=SimpleNamespace( + download_required_models=failing, install_environment=failing)) + with ( + patch.object(self.ai_setup, "ai_status", return_value={"ready": False, "environment_ready": ready}), + patch.object(self.ai_setup.bpy, "ops", ops, create=True), + ): + self.assertEqual(operator.execute(None), {"CANCELLED"}) + operator.report.assert_called_once_with({"ERROR"}, "setup failed") + owner = SimpleNamespace(report=Mock()) + with ( + patch.object(self.ai, "ai_ready", return_value=False), + patch.object(self.ai.bpy, "ops", SimpleNamespace(anyimage=SimpleNamespace( + setup_ai_environment=Mock(side_effect=RuntimeError("Error: setup failed\n")))), create=True), + ): + self.assertEqual(self.ai.invoke_ai_setup_if_needed(owner), {"CANCELLED"}) + owner.report.assert_called_once_with({"ERROR"}, "setup failed") + + def test_depth_and_relief_cleanup_on_nested_failure(self): + module = self.convert_to_plane + for plane_type in ("DEPTH", "RELIEF"): + for failure in (RuntimeError("Error: invalid input\n"), None): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "prepared" / "input.png" + path.parent.mkdir() + path.touch() + operator = SimpleNamespace(mesh_detail=4, report=Mock()) + context = SimpleNamespace(object=SimpleNamespace(name="source", as_pointer=lambda: 1, + data=SimpleNamespace(as_pointer=lambda: 2)), scene=object()) + generate = Mock(side_effect=failure, return_value={"CANCELLED"}) + with ( + patch.object(module, "invoke_ai_setup_if_needed", return_value=None), + patch.object(module, "prepare_material_color_input", return_value=path), + patch.object(module, "material_analysis_input", return_value=path), + patch.object(module, "depth_plane_settings", return_value=("model", 5)), + patch.object(module, "configured_max_ai_input_size", return_value=2048), + patch.object(module.bpy, "ops", SimpleNamespace(anyimage=SimpleNamespace(generate_depth_plane=generate)), create=True), + ): + self.assertEqual(module.start_depth_plane_job(operator, context, plane_type), {"CANCELLED"}) + self.assertFalse(path.parent.exists()) diff --git a/tests/anyimage/operators/test_clipboard_cache.py b/tests/anyimage/operators/test_clipboard_cache.py new file mode 100644 index 0000000..859181f --- /dev/null +++ b/tests/anyimage/operators/test_clipboard_cache.py @@ -0,0 +1,113 @@ +"""Verify cache identity against edits to real Blender images.""" + +from io import BytesIO +from types import SimpleNamespace + +import bpy +import numpy as np +import pytest +from PIL import Image + +from anyimage.common import image as images +from anyimage.common.selection import SelectionMask +from anyimage.operators.clipboard_image import actions +from anyimage.operators.mask_tool import EditImageAlpha + + +@pytest.fixture +def clipboard(): + bpy.ops.wm.read_factory_settings(use_empty=True) + stream = BytesIO() + Image.fromarray(np.array([[[100, 150, 200, 255], [50, 80, 120, 0]]], np.uint8)).save(stream, format="PNG") + yield stream.getvalue() + bpy.ops.wm.read_factory_settings(use_empty=True) + + +@pytest.mark.parametrize("edit", ["paint", "packed", "hidden_rgb", "size", "space", "alpha", "legacy", "unreadable"]) +def test_changed_cache_is_skipped_without_modifying_candidate(clipboard, edit, monkeypatch): + original, reused = actions.acquire_packed_image(clipboard, ".png") + expected = images.image_pixels(original) + assert not reused + assert actions.acquire_packed_image(clipboard, ".png") == (original, True) + if edit in {"paint", "packed"}: + original.pixels[0] = 1 + if edit == "packed": + original.pack() + elif edit == "hidden_rgb": + original.pixels[4] = 1 + elif edit == "size": + original.scale(4, 2) + elif edit == "space": + original.colorspace_settings.name = "Non-Color" + elif edit == "alpha": + original.alpha_mode = "PREMUL" + elif edit == "legacy": + del original[actions.CLIPBOARD_CONTENT_PROPERTY] + else: + read = actions._clipboard_content_hash + def fail_candidate(image): + if image == original: + raise RuntimeError("Unreadable candidate") + return read(image) + monkeypatch.setattr(actions, "_clipboard_content_hash", fail_candidate) + state = images.image_content_state(original) + current_pixels = original.pixels[:] + replacement, reused = actions.acquire_packed_image(clipboard, ".png") + assert replacement != original and not reused + np.testing.assert_array_equal(images.image_pixels(replacement), expected) + assert (original.colorspace_settings.name, original.alpha_mode) == (state["colorspace"], state["alpha_mode"]) + assert bytes(original.packed_file.data) == state["packed"] + np.testing.assert_array_equal(original.pixels[:], current_pixels) + + +@pytest.mark.parametrize("shared", [False, True]) +def test_mask_edit_then_paste_preserves_original_clipboard_content(clipboard, shared): + original, _ = actions.acquire_packed_image(clipboard, ".png") + expected = images.image_pixels(original) + owner = bpy.data.objects.new("Clipboard Empty", None) + owner.empty_display_type = "IMAGE" + owner.data = original + bpy.context.collection.objects.link(owner) + if shared: + other = owner.copy() + bpy.context.collection.objects.link(other) + mask = SelectionMask(np.full((1, 2), .5, np.float32), (0, 0, 2, 1)) + assert EditImageAlpha._apply_mask(SimpleNamespace(mode="SUBTRACT"), bpy.context, owner, mask) == {"FINISHED"} + edited = owner.data + edited_pixels = images.image_pixels(edited) + assert edited.packed_file + assert (edited == original) == (not shared) + pasted, reused = actions.acquire_packed_image(clipboard, ".png") + assert reused == shared + assert pasted != edited + np.testing.assert_array_equal(images.image_pixels(pasted), expected) + np.testing.assert_array_equal(images.image_pixels(edited), edited_pixels) + + +def test_painted_clipboard_plane_keeps_reference_node_and_brush_cache(clipboard, monkeypatch): + original, _ = actions.acquire_packed_image(clipboard, ".png") + expected = images.image_pixels(original) + monkeypatch.setattr(actions, "_place_in_view", lambda *_: None) + monkeypatch.setattr(actions, "_select_only", lambda *_: None) + obj = actions.add_image_plane(bpy.context, original, shadeless=False) + color = next(n.image for n in obj.data.materials[0].node_tree.nodes if n.type == "TEX_IMAGE") + assert color != original and actions.CLIPBOARD_HASH_PROPERTY not in color + assert actions.CLIPBOARD_CONTENT_PROPERTY not in color + color.pixels[0] = 1 + color.pack() + for _ in range(3): + pasted, reused = actions.acquire_packed_image(clipboard, ".png") + assert pasted == original and reused + np.testing.assert_array_equal(images.image_pixels(pasted), expected) + + +def test_failed_clipboard_plane_placement_releases_created_data(clipboard, monkeypatch): + original, _ = actions.acquire_packed_image(clipboard, ".png") + before = {name: set(getattr(bpy.data, name)) for name in ("images", "objects", "meshes", "materials")} + def fail(*_): + raise RuntimeError("Placement failed") + monkeypatch.setattr(actions, "_place_in_view", fail) + with pytest.raises(RuntimeError, match="Placement failed"): + actions.add_image_plane(bpy.context, original) + for name, items in before.items(): + assert set(getattr(bpy.data, name)) == items diff --git a/tests/anyimage/operators/test_clipboard_image.py b/tests/anyimage/operators/test_clipboard_image.py index 31f2dde..03b6521 100644 --- a/tests/anyimage/operators/test_clipboard_image.py +++ b/tests/anyimage/operators/test_clipboard_image.py @@ -1,5 +1,6 @@ from tests.support.paths import PROJECT_ROOT import importlib +from contextlib import nullcontext import struct import sys import tempfile @@ -400,13 +401,14 @@ class ClipboardImageTest(unittest.TestCase): for shadeless in (False, True): with ( patch.object(self.actions, "create_image_material", return_value="Material") as create_material, + patch.object(self.actions, "material_color_image", return_value=nullcontext("Color")), patch.object(image_plane, "create_image_plane_object") as create_plane, patch.object(self.actions, "_place_in_view"), patch.object(self.actions, "_select_only"), ): self.actions.add_image_plane(context, image, shadeless=shadeless) create_material.assert_called_once_with( - image, image, shadeless=shadeless, scene=context.scene, + image, "Color", shadeless=shadeless, scene=context.scene, ) self.assertEqual(create_plane.call_args.args[3], "Material") @@ -570,10 +572,11 @@ class ClipboardImageTest(unittest.TestCase): images=[matching_image, unpacked_image], ) try: - self.assertIs( - self.actions._find_packed_clipboard_image("matching-hash"), - matching_image, - ) + with patch.object(matching_image, "get", side_effect=lambda key: "matching-hash" if key == self.actions.CLIPBOARD_HASH_PROPERTY else "pixels"), patch.object(self.actions, "_clipboard_content_hash", return_value="pixels"): + self.assertIs( + self.actions._find_packed_clipboard_image("matching-hash"), + matching_image, + ) self.assertIsNone( self.actions._find_packed_clipboard_image("unpacked-hash") ) @@ -586,10 +589,11 @@ class ClipboardImageTest(unittest.TestCase): original_data = self.actions.bpy.data self.actions.bpy.data = SimpleNamespace(images=images) try: - loaded_image, reused = self.actions.acquire_packed_image( - b"image-data", - ".png", - ) + with patch.object(self.actions, "_clipboard_content_hash", return_value="pixels"): + loaded_image, reused = self.actions.acquire_packed_image( + b"image-data", + ".png", + ) finally: self.actions.bpy.data = original_data diff --git a/tests/anyimage/operators/test_conversion_source_images.py b/tests/anyimage/operators/test_conversion_source_images.py new file mode 100644 index 0000000..b008cdf --- /dev/null +++ b/tests/anyimage/operators/test_conversion_source_images.py @@ -0,0 +1,185 @@ +"""Verify conversion ownership, source recovery, and persistent color data.""" + +from types import SimpleNamespace + +import bpy +import numpy as np +import pytest +from PIL import Image + +from anyimage.common import image as images, material +from anyimage.common.color_image import prepare_material_color_input, cleanup_material_color_input +from anyimage.operators.convert_to_plane import operators, object as conversion, image_plane + + +@pytest.fixture +def source(): + bpy.ops.wm.read_factory_settings(use_empty=True) + image = bpy.data.images.new("Source.png", width=2, height=1, alpha=True) + image.pixels.foreach_set([0.2, 0.4, 0.8, 0.25, 0.6, 0.3, 0.1, 0]) + obj = bpy.data.objects.new("Source", None) + obj.empty_display_type = "IMAGE" + obj.data = image + bpy.context.collection.objects.link(obj) + obj.select_set(True) + bpy.context.view_layer.objects.active = obj + yield obj + bpy.ops.wm.read_factory_settings(use_empty=True) + + +def share_image(source, sharing): + if sharing in {"visible", "hidden", "other_scene", "same_path"}: + other = source.copy() + if sharing == "same_path": + source.data.filepath_raw = "//source.png" + other.data = source.data.copy() + collection = bpy.context.collection + if sharing == "other_scene": + collection = bpy.data.scenes.new("Other scene").collection + collection.objects.link(other) + other.hide_viewport = sharing == "hidden" + elif sharing == "material": + existing = bpy.data.materials.new("Existing") + existing.use_nodes = True + existing.node_tree.nodes.new("ShaderNodeTexImage").image = source.data + + +def run_conversion(source, entry, monkeypatch): + if entry == "PLANE": + operators.ConvertToPlane.execute(SimpleNamespace(mesh_detail=0), bpy.context) + else: + metadata = {"image_size": (2, 1), "reference_depth": 1, + "intrinsics": ((2, 0, 1), (0, 2, 0.5), (0, 0, 1))} + monkeypatch.setattr(conversion, "load_depth_metadata", lambda _: metadata) + def load_depth(*_): + image = bpy.data.images.new("Depth", width=2, height=1, float_buffer=True) + image.colorspace_settings.name = "Non-Color" + return image + monkeypatch.setattr(conversion, "load_depth_result_image", load_depth) + monkeypatch.setattr(conversion, "load_normal_result_image", load_depth) + monkeypatch.setattr(conversion, "bpy", SimpleNamespace(data=bpy.data, + ops=SimpleNamespace(ed=SimpleNamespace(undo_push=lambda **_: {"FINISHED"})))) + path = prepare_material_color_input(source.data) + try: + conversion.create_depth_plane_from_result(bpy.context, SimpleNamespace(file=lambda key: key), + SimpleNamespace(source_object_name=source.name, source_identity=str(source.as_pointer()), + image_identity=str(source.data.as_pointer()), color_path=str(path), plane_type=entry, mesh_detail=0)) + finally: + cleanup_material_color_input(path) + return bpy.context.object + + +@pytest.mark.parametrize("entry", ["PLANE", "DEPTH", "RELIEF"]) +@pytest.mark.parametrize("sharing", ["none", "visible", "hidden", "other_scene", "material", "same_path"]) +def test_conversion_selects_source_by_empty_identity(source, monkeypatch, entry, sharing): + share_image(source, sharing) + original = source.data + before = images.image_pixels(original) + name, path = original.name, original.filepath_raw + monkeypatch.setattr(material, "configured_material_view_adaptation", lambda: False) + obj = run_conversion(source, entry, monkeypatch) + color = next(n.image for n in obj.data.materials[0].node_tree.nodes if n.type == "TEX_IMAGE") + assert color != original + assert color.name == "Source_color.png" + assert color.filepath_raw != original.filepath_raw + assert original.name == name and original.filepath_raw == path + assert color.colorspace_settings.name == "sRGB" and color.alpha_mode == "PREMUL" + assert original.alpha_mode == "STRAIGHT" + np.testing.assert_array_equal(images.image_pixels(original), before) + np.testing.assert_allclose(images.image_pixels(color), before, atol=1 / 255) + if sharing == "material": + existing = bpy.data.materials["Existing"] + assert next(n.image for n in existing.node_tree.nodes if n.type == "TEX_IMAGE") == original + + +@pytest.mark.parametrize("shared", [False, True]) +@pytest.mark.parametrize("failure", ["material", "modifier", "finalize"]) +@pytest.mark.parametrize("entry", ["PLANE", "DEPTH", "RELIEF"]) +def test_conversion_failure_restores_image_and_releases_resources(source, monkeypatch, shared, failure, entry): + if shared: + share_image(source, "visible") + source.data.pack() + source.data.pixels[0] = 0.75 + original = source.data + state = images.image_content_state(original) + before = {name: set(getattr(bpy.data, name)) for name in ("images", "objects", "meshes", "materials")} + def fail(*args, **kwargs): + raise RuntimeError("Conversion failed") + monkeypatch.setattr(material, "configured_material_view_adaptation", lambda: True) + if failure == "material": + monkeypatch.setattr(material, "material_node_group", fail) + elif failure == "finalize": + monkeypatch.setattr(conversion, "finalize_object_result", fail) + elif entry == "PLANE": + monkeypatch.setattr(image_plane, "add_image_plane_processing", fail) + else: + monkeypatch.setattr(conversion, "set_modifier_input", fail) + with pytest.raises(RuntimeError, match="Conversion failed"): + run_conversion(source, entry, monkeypatch) + for name, values in before.items(): + assert set(getattr(bpy.data, name)) == values + assert source.data == original + assert (original.colorspace_settings.name, original.alpha_mode) == (state["colorspace"], state["alpha_mode"]) + assert bytes(original.packed_file.data) == state["packed"] + np.testing.assert_array_equal(original.pixels[:], state["pixels"]) + + +@pytest.mark.parametrize("shared", [False, True]) +@pytest.mark.parametrize("kind", ["generated", "file", "dirty", "packed"]) +def test_conversion_pixels_survive_library_reload(source, tmp_path, monkeypatch, kind, shared): + if kind in {"file", "dirty"}: + path = tmp_path / "source.png" + Image.fromarray(np.array([[[51, 102, 204, 64], [153, 76, 25, 0]]], np.uint8)).save(path) + source.data = bpy.data.images.load(str(path)) + if kind == "dirty": + source.data.pixels[0] = 0.8 + elif kind == "packed": + source.data.pack() + source.data.reload() + if shared: + share_image(source, "visible") + expected = images.image_pixels(source.data) + monkeypatch.setattr(material, "configured_material_view_adaptation", lambda: True) + obj = run_conversion(source, "PLANE", monkeypatch) + color = next(n.image for n in obj.data.materials[0].node_tree.nodes if n.type == "TEX_IMAGE") + library = tmp_path / "converted.blend" + bpy.data.libraries.write(str(library), {color}) + with bpy.data.libraries.load(str(library)) as (saved, loaded): + loaded.images = saved.images + restored = loaded.images[0] + assert (restored.colorspace_settings.name, restored.alpha_mode) == (color.colorspace_settings.name, color.alpha_mode) + np.testing.assert_allclose(images.image_pixels(restored), expected, atol=1 / 255) + + +@pytest.mark.parametrize("entry", ["DEPTH", "RELIEF"]) +def test_depth_result_rejects_replaced_source_image(source, entry): + operator = SimpleNamespace(source_object_name=source.name, source_identity=str(source.as_pointer()), + image_identity=str(source.data.as_pointer()), plane_type=entry) + source.data = source.data.copy() + with pytest.raises(ValueError, match="source image has changed"): + conversion.create_depth_plane_from_result(bpy.context, None, operator) + + +@pytest.mark.parametrize("shared", [False, True]) +def test_plane_undo_redo_restores_image_configuration(source, monkeypatch, shared): + if shared: + share_image(source, "visible") + original_name = source.data.name + expected = images.image_pixels(source.data) + monkeypatch.setattr(material, "configured_material_view_adaptation", lambda: False) + bpy.context.preferences.edit.use_global_undo = True + bpy.utils.register_class(operators.ConvertToPlane) + try: + bpy.ops.ed.undo_push(message="Before Plane") + assert bpy.ops.anyimage.convert_to_plane("EXEC_DEFAULT", True) == {"FINISHED"} + color_name = "Source_color.png" + assert bpy.data.images[color_name].alpha_mode == "PREMUL" + assert bpy.ops.ed.undo() == {"FINISHED"} + assert bpy.data.objects["Source"].type == "EMPTY" + assert bpy.data.images[original_name].alpha_mode == "STRAIGHT" + np.testing.assert_allclose(images.image_pixels(bpy.data.images[original_name]), expected, atol=1 / 255) + assert bpy.ops.ed.redo() == {"FINISHED"} + assert bpy.data.objects["Source"].type == "MESH" + assert bpy.data.images[color_name].alpha_mode == "PREMUL" + finally: + bpy.utils.unregister_class(operators.ConvertToPlane) diff --git a/tests/anyimage/operators/test_cutout_objects.py b/tests/anyimage/operators/test_cutout_objects.py index ce80b92..765689e 100644 --- a/tests/anyimage/operators/test_cutout_objects.py +++ b/tests/anyimage/operators/test_cutout_objects.py @@ -7,6 +7,8 @@ from mathutils import Matrix, Vector from anyimage.common import depth as depth_data +from anyimage.common import color_image as colors +from tests.support.color_image import color_image as prepared_color_image from anyimage.common import image as image_data from anyimage.common import material as material_data from anyimage.common import object as object_data @@ -808,24 +810,20 @@ def test_bounds_input_preserves_rectangular_rgba_and_stays_packed_after_cleanup( ), dtype=np.float32, ) - path = image_data.prepare_image_bounds_input(image, rgba, (1, 1, 5, 4)) + path = colors.prepare_material_color_input(image, (1, 1, 5, 4), rgba) result = None try: - result = image_data.load_packed_rgba_image( - path, - image, - "Bounds Result", - ) + result = colors.load_material_color_image(path) loaded = image_data.image_rgba(result) np.testing.assert_allclose(loaded, rgba[1:4, 1:5], atol=1.0 / 255.0) assert result.packed_file is not None - image_data.cleanup_image_input(path, True) + colors.cleanup_material_color_input(path) assert not path.exists() assert result.packed_file is not None finally: - image_data.cleanup_image_input(path, True) + colors.cleanup_material_color_input(path) if result is not None: bpy.data.images.remove(result, do_unlink=True) @@ -898,8 +896,8 @@ def test_local_cutout_builds_from_blender_image_pixels_without_a_job(): selection_mask.values, alpha_threshold=0.5, ) - color_image = cutout_object.create_cropped_color_image( - source, + color_image = prepared_color_image( + source.data, selection_mask.bounds, rgba, ) diff --git a/tests/anyimage/operators/test_cutout_operators.py b/tests/anyimage/operators/test_cutout_operators.py index bb4905c..e410acc 100644 --- a/tests/anyimage/operators/test_cutout_operators.py +++ b/tests/anyimage/operators/test_cutout_operators.py @@ -1,3 +1,4 @@ +from contextlib import nullcontext import importlib import json from pathlib import Path @@ -69,8 +70,8 @@ class CutoutOperatorsTest(BlenderTestCase): ), patch.object( self.cutout_main, - "create_cropped_color_image", - return_value="Color", + "material_color_image", + return_value=nullcontext("Color"), ), patch.object( self.cutout_main, "create_cutout_shape" @@ -163,9 +164,10 @@ class CutoutOperatorsTest(BlenderTestCase): ), patch.object( self.cutout_main, - "prepare_image_bounds_input", + "prepare_material_color_input", return_value=Path(__file__), ) as prepare_input, + patch.object(self.cutout_main, "material_analysis_input", return_value=Path(__file__)), patch.object( self.cutout_main.JobOperatorBase, "execute", @@ -179,8 +181,8 @@ class CutoutOperatorsTest(BlenderTestCase): rasterize.assert_called_once() prepare_input.assert_called_once_with( source.data, - source_rgba, selection_mask.bounds, + source_rgba, ) @@ -244,6 +246,7 @@ class CutoutOperatorsTest(BlenderTestCase): operator.alpha_threshold = 0.25 operator.fine_outline = False operator.input_path = __file__ + operator.color_path = __file__ operator.source_object_name = "Source" operator.shape = "DEPTH_SOLID" operator.model = self.anyimage.properties.DEFAULT_MOGE2_MODEL_KEY @@ -256,10 +259,10 @@ class CutoutOperatorsTest(BlenderTestCase): with ( patch.object(self.cutout_main, "require_environment"), patch.object(self.cutout_main, "require_model") as require_model, + patch.object(self.cutout_main, "require_input_path", return_value=Path("input.png")), patch.object( self.cutout_main, "production_device", return_value="CPU" ), - patch.object(self.cutout_main, "source_start_frame", return_value=1), patch.object( self.cutout_main, "moge2_parameters", @@ -372,6 +375,7 @@ class CutoutOperatorsTest(BlenderTestCase): operator.source_object_name = "Source" operator.selection_bounds_json = "[10,20,14,23]" operator.input_path = __file__ + operator.color_path = __file__ operator.shape = "FLAT" operator.point_spacing = 32 operator.generate_normal = True @@ -399,7 +403,6 @@ class CutoutOperatorsTest(BlenderTestCase): color_result.assert_called_once_with( __file__, - source, selection_mask.bounds, ) diff --git a/tests/anyimage/operators/test_hdr_background.py b/tests/anyimage/operators/test_hdr_background.py new file mode 100644 index 0000000..51adb61 --- /dev/null +++ b/tests/anyimage/operators/test_hdr_background.py @@ -0,0 +1,115 @@ +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import bpy +import numpy as np +import pytest + +from anyimage.common.hdr_image import HdrBackgroundInput +from anyimage.common.image import cleanup_image_input, image_rgba +from anyimage.common.image_target import ImageEditTarget +from anyimage.operators import remove_background as operators +from anyimage.runtime import JobOperatorBase +from tests.support.hdr_image import hdr_texture +from tests.support.image_texture import texture, texture_context + + +@pytest.mark.parametrize("owner_kind", ["texture", "empty"]) +@pytest.mark.parametrize("shared", [False, True]) +def test_hdr_background_undo_redo(hdr_texture, tmp_path, owner_kind, shared): + node = hdr_texture + source = node.image + before = image_rgba(source).copy() + if owner_kind == "empty": + owner = bpy.data.objects.new("HDR Empty", None) + owner.empty_display_type = "IMAGE" + owner.data = source + bpy.context.scene.collection.objects.link(owner) + node.image = source if shared else None + elif shared: + other = node.id_data.nodes.new("ShaderNodeTexImage") + other.image = source + alpha_path = tmp_path / "alpha.npy" + np.save(alpha_path, np.full((3, 4), 0.37123, np.float32)) + + def current_owner(): + if owner_kind == "empty": + return bpy.data.objects["HDR Empty"] + return bpy.data.materials["Texture material"].node_tree.nodes["Source texture"] + + def current_image(): + owner = current_owner() + return owner.data if owner_kind == "empty" else owner.image + + def execute(operator, context): + owner = current_owner() + target_context = SimpleNamespace(object=owner, space_data=None) if owner_kind == "empty" else texture_context(owner) + operator._image_target = ImageEditTarget.capture(target_context) + operator._hdr_input = HdrBackgroundInput.prepare(current_image()) + operator.input_path, operator.delete_input = str(operator._hdr_input.path), True + try: + operator.response(context, SimpleNamespace(file=lambda key: alpha_path)) + finally: + operator.cleanup() + return {"FINISHED"} + + operation = operators.RunBackgroundRemoval + bpy.context.preferences.edit.use_global_undo = True + with patch.object(operation, "execute", execute), patch.object(operation, "poll", classmethod(lambda cls, context: True), create=True): + bpy.utils.register_class(operation) + try: + bpy.ops.ed.undo_push(message="Before HDR removal") + assert bpy.ops.anyimage.run_background_removal("EXEC_DEFAULT", True) == {"FINISHED"} + after = image_rgba(current_image()).copy() + np.testing.assert_array_equal(after[..., :3], before[..., :3]) + np.testing.assert_allclose(after[..., 3], before[..., 3] * np.float32(0.37123), atol=1e-7) + assert bpy.ops.ed.undo() == {"FINISHED"} + np.testing.assert_array_equal(image_rgba(current_image()), before) + assert bpy.ops.ed.redo() == {"FINISHED"} + np.testing.assert_array_equal(image_rgba(current_image()), after) + assert current_image().is_float + finally: + bpy.utils.unregister_class(operation) + + +@pytest.mark.parametrize("outcome", ["cancel", "raise"]) +def test_hdr_submission_failure_releases_preview_and_snapshot(hdr_texture, outcome): + target = ImageEditTarget.capture(texture_context(hdr_texture)) + paths = [] + + def submit(operator, context): + paths.append(Path(operator.input_path)) + assert paths[-1].is_file() + if outcome == "raise": + raise RuntimeError("Submission failed") + return {"CANCELLED"} + + # Use the registered type so super().execute follows the real Operator inheritance. + operation = operators.RunBackgroundRemoval + with patch.object(ImageEditTarget, "capture", return_value=target), patch.object(JobOperatorBase, "execute", submit): + bpy.utils.register_class(operation) + try: + if outcome == "raise": + with pytest.raises(RuntimeError, match="Submission failed"): + bpy.ops.anyimage.run_background_removal("EXEC_DEFAULT") + else: + assert bpy.ops.anyimage.run_background_removal("EXEC_DEFAULT") == {"CANCELLED"} + finally: + bpy.utils.unregister_class(operation) + assert paths and not paths[0].parent.exists() + + +def test_hdr_request_uses_alpha_output_and_current_still(hdr_texture): + operation = operators.RunBackgroundRemoval + snapshot = HdrBackgroundInput.prepare(hdr_texture.image) + operator = SimpleNamespace(input_path=str(snapshot.path), _hdr_input=snapshot, + _image_target=SimpleNamespace(start_frame=17)) + context = SimpleNamespace(scene=SimpleNamespace(anyimage_settings=SimpleNamespace(video_frames=12))) + try: + with patch.object(operators, "require_environment"), patch.object(operators, "require_model"), patch.object(operators, "production_device", return_value="CPU"): + parameters = operation.request(operator, context) + assert parameters["output_kind"] == "alpha" + assert parameters["start_frame"] == parameters["video_frames"] == 1 + finally: + cleanup_image_input(snapshot.path, True) diff --git a/tests/anyimage/operators/test_native_ai_failures.py b/tests/anyimage/operators/test_native_ai_failures.py new file mode 100644 index 0000000..b5c995a --- /dev/null +++ b/tests/anyimage/operators/test_native_ai_failures.py @@ -0,0 +1,77 @@ +from unittest.mock import patch + +import bpy +import pytest + +from anyimage.operators import ai_setup, remove_background +from anyimage.operators.convert_to_plane import operators as plane + + +@pytest.mark.parametrize("kind", ["background", "depth", "relief"]) +def test_nested_native_operator_preserves_plain_error(kind, tmp_path, capfd): + reason = "Unsupported input format: .tiff" + if kind == "background": + inner, outer = remove_background.RunBackgroundRemoval, remove_background.RemoveImageBackground + else: + inner = plane.GenerateDepthPlane + outer = plane.ConvertToDepthPlane if kind == "depth" else plane.ConvertToReliefPlane + + def reject(operator, context): + operator.report({"ERROR"}, reason) + return {"CANCELLED"} + + prepared = tmp_path / "prepared" + prepared.mkdir() + path = prepared / "input.png" + path.touch() + with ( + patch.object(inner, "execute", reject), + patch.object(outer, "poll", classmethod(lambda cls, context: True)), + patch.object(plane, "invoke_ai_setup_if_needed", return_value=None), + patch.object(plane, "prepare_material_color_input", return_value=path), + patch.object(plane, "material_analysis_input", return_value=path), + patch.object(plane, "depth_plane_settings", return_value=("model", 5)), + patch.object(plane, "configured_max_ai_input_size", return_value=2048), + ): + owner = bpy.data.objects.new("Native validation source", None) + image = bpy.data.images.new("Native validation image", width=1, height=1) + owner.empty_display_type = "IMAGE" + owner.data = image + bpy.context.scene.collection.objects.link(owner) + bpy.context.view_layer.objects.active = owner + bpy.utils.register_class(inner) + bpy.utils.register_class(outer) + try: + with pytest.raises(RuntimeError) as error: + getattr(bpy.ops.anyimage, outer.bl_idname.split(".")[1])() + assert str(error.value).strip() == f"Error: {reason}" + assert "Traceback" not in capfd.readouterr().err + if kind != "background": + assert not prepared.exists() + finally: + bpy.utils.unregister_class(outer) + bpy.utils.unregister_class(inner) + bpy.data.objects.remove(owner, do_unlink=True) + bpy.data.images.remove(image) + + +def test_native_setup_operator_preserves_plain_error(capfd): + class RejectInstall(bpy.types.Operator): + bl_idname = "anyimage.install_environment" + bl_label = "Reject Install" + + def execute(self, context): + self.report({"ERROR"}, "Setup unavailable") + return {"CANCELLED"} + + with patch.object(ai_setup, "ai_status", return_value={"ready": False, "environment_ready": False}): + bpy.utils.register_class(RejectInstall) + bpy.utils.register_class(ai_setup.SetupAIEnvironment) + try: + with pytest.raises(RuntimeError) as error: + bpy.ops.anyimage.setup_ai_environment(use_mirror=False) + assert str(error.value).strip() == "Error: Setup unavailable" + assert "Traceback" not in capfd.readouterr().err + finally: + bpy.utils.unregister_class(ai_setup.SetupAIEnvironment) + bpy.utils.unregister_class(RejectInstall) diff --git a/tests/anyimage/operators/test_panorama_conversion.py b/tests/anyimage/operators/test_panorama_conversion.py new file mode 100644 index 0000000..bac07a5 --- /dev/null +++ b/tests/anyimage/operators/test_panorama_conversion.py @@ -0,0 +1,188 @@ +"""Verify the public panorama entry and transactional Blender result handling.""" + +import json +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import bpy +import numpy as np +import pytest +from PIL import Image +from mathutils import Matrix + +from anyimage.operators.convert_to_panorama import operators, object as conversion +from server.geometry.depth_texture import write_float_exr +from anyimage.common.color_image import prepare_material_color_input, cleanup_material_color_input + + +@pytest.fixture +def source(): + bpy.ops.wm.read_factory_settings(use_empty=True) + image = bpy.data.images.new("Current panorama", width=32, height=16, alpha=True) + rgba = np.ones((16, 32, 4), np.float32) + rgba[:8, :, 3] = 0 + rgba[8:12, :, 3] = 0.5 + image.pixels.foreach_set(rgba.ravel()) + obj = bpy.data.objects.new("Panorama Empty", None) + obj.empty_display_type = "IMAGE" + obj.data = image + bpy.context.collection.objects.link(obj) + obj.select_set(True) + bpy.context.view_layer.objects.active = obj + return obj + + +def test_entry_submits_current_alpha_preferences_and_cleans_cancelled_input(source, monkeypatch): + submitted = {} + def submit(**parameters): + submitted.update(parameters) + with Image.open(parameters["input_path"]) as image: + alpha = np.asarray(image)[..., 3] + assert not alpha[8:].any() + assert np.all(abs(alpha[4:8].astype(float) - 128) <= 1) + return {"CANCELLED"} + monkeypatch.setattr(operators, "invoke_ai_setup_if_needed", lambda _operator: None) + monkeypatch.setattr(operators, "addon_preferences", lambda _: SimpleNamespace(geometry_model="MOGE2_VITB", geometry_resolution_level=2)) + monkeypatch.setattr(operators, "configured_max_ai_input_size", lambda _: 1024) + monkeypatch.setattr(operators, "bpy", SimpleNamespace(ops=SimpleNamespace(anyimage=SimpleNamespace(generate_panorama=submit)))) + operator = SimpleNamespace(mesh_detail=4, report=Mock()) + assert operators.ConvertToPanorama.execute(operator, bpy.context) == {"CANCELLED"} + assert submitted["model"] == "MOGE2_VITB" and submitted["resolution_level"] == 2 + assert submitted["max_input_size"] == 1024 and submitted["mesh_detail"] == 4 + assert submitted["color_path"] == submitted["input_path"] + from pathlib import Path + assert not Path(submitted["input_path"]).exists() + assert bpy.data.objects.get(source.name) == source + + +@pytest.mark.parametrize("image_source,size", [("MOVIE", (32, 16)), ("SEQUENCE", (32, 16)), ("FILE", (32, 32))]) +def test_invalid_input_is_rejected_before_setup(image_source, size, monkeypatch): + setup = Mock() + monkeypatch.setattr(operators, "invoke_ai_setup_if_needed", setup) + obj = SimpleNamespace(data=SimpleNamespace(source=image_source, size=size)) + assert operators.ConvertToPanorama.execute(SimpleNamespace(report=Mock()), SimpleNamespace(object=obj)) == {"CANCELLED"} + setup.assert_not_called() + + +@pytest.fixture +def artifacts(tmp_path, source): + rgba = np.ones((16, 32, 4), np.float32) + rgba[..., :3] = 3 + write_float_exr(rgba, tmp_path / "depth.exr") + (tmp_path / "depth.json").write_text(json.dumps({"projection": "equirectangular", "image_size": [32, 16], "reference_distance": 3})) + result = SimpleNamespace(directory=tmp_path, file=lambda key: tmp_path / {"depth": "depth.exr", "depth_metadata": "depth.json"}[key]) + operator = SimpleNamespace(source_object_name=source.name, source_identity=str(source.as_pointer()), + image_identity=str(source.data.as_pointer()), mesh_detail=2, + color_path=str(prepare_material_color_input(source.data))) + yield result, operator + cleanup_material_color_input(operator.color_path) + + +def test_result_replaces_source_with_zero_rotation_and_shadeless_material(source, artifacts, monkeypatch): + result, operator = artifacts + matrix = Matrix.Translation((2, -3, 4)) @ Matrix.Rotation(0.6, 4, "Y") @ Matrix.Diagonal((2, 3, 4, 1)) + source.matrix_world = matrix + source.empty_image_offset = (0.3, -0.8) + undo = Mock(return_value={"FINISHED"}) + monkeypatch.setattr(conversion, "bpy", SimpleNamespace(data=bpy.data, ops=SimpleNamespace(ed=SimpleNamespace(undo_push=undo)))) + obj = conversion.create_panorama_from_result(bpy.context, result, operator) + assert obj.name == operator.source_object_name and obj.type == "MESH" + np.testing.assert_allclose(obj.location, matrix.translation, atol=1e-6) + np.testing.assert_allclose(obj.scale, matrix.to_scale(), atol=1e-6) + np.testing.assert_array_equal(obj.rotation_euler, (0, 0, 0)) + material = obj.data.materials[0] + assert any(n.type == "GROUP" and n.node_tree.name == "O Shadeless" for n in material.node_tree.nodes) + assert [n.node_tree.name for n in material.node_tree.nodes if n.type == "GROUP"] == ["O Shadeless"] + texture = next(n for n in material.node_tree.nodes if n.type == "TEX_IMAGE") + assert texture.extension == "REPEAT" + assert bpy.context.view_layer.objects.active == obj + assert all(image.packed_file for image in bpy.data.images if image.name.endswith(("_color.png", "_depth.exr"))) + graph = bpy.context.evaluated_depsgraph_get() + mesh = obj.evaluated_get(graph).to_mesh() + try: + assert len(mesh.polygons) == 96 and mesh.uv_layers.active + assert mesh.materials[0].original == material + assert all(face.material_index == 0 and face.use_smooth for face in mesh.polygons) + finally: + obj.evaluated_get(graph).to_mesh_clear() + undo.assert_called_once_with(message="Convert to Panorama") + + +@pytest.mark.parametrize("shared", [False, True]) +def test_panorama_preserves_hdr_pixels_color_space_and_alpha(source, artifacts, monkeypatch, shared): + result, operator = artifacts + hdr = bpy.data.images.new("Original HDR", width=32, height=16, alpha=True, float_buffer=True) + hdr.alpha_mode = "PREMUL" + rgba = np.ones((16, 32, 4), np.float32) + rgba[..., :3] = (4.0, 2.0, 0.5) + rgba[..., 3] = 0.5 + hdr.pixels.foreach_set(rgba.ravel()) + source.data = hdr + operator.image_identity = str(hdr.as_pointer()) + cleanup_material_color_input(operator.color_path) + operator.color_path = str(prepare_material_color_input(hdr)) + if shared: + other = source.copy() + bpy.context.collection.objects.link(other) + original_space, original_alpha = hdr.colorspace_settings.name, hdr.alpha_mode + monkeypatch.setattr(conversion, "bpy", SimpleNamespace(data=bpy.data, ops=SimpleNamespace(ed=SimpleNamespace(undo_push=lambda **kw: {"FINISHED"})) )) + obj = conversion.create_panorama_from_result(bpy.context, result, operator) + color = next(n.image for n in obj.data.materials[0].node_tree.nodes if n.type == "TEX_IMAGE") + assert color != hdr + assert color.is_float and color.packed_file + assert (color.colorspace_settings.name, color.alpha_mode) == (original_space, original_alpha) + np.testing.assert_allclose(color.pixels[:], rgba.ravel(), atol=1e-6) + assert (hdr.colorspace_settings.name, hdr.alpha_mode) == (original_space, original_alpha) + library = result.directory / "hdr-copy.blend" + bpy.data.libraries.write(str(library), {color}) + with bpy.data.libraries.load(str(library)) as (saved, loaded): + loaded.images = saved.images + restored = loaded.images[0] + assert restored.is_float + assert (restored.colorspace_settings.name, restored.alpha_mode) == (original_space, original_alpha) + np.testing.assert_allclose(restored.pixels[:], rgba.ravel(), atol=1e-6) + + +@pytest.mark.parametrize("failure", ["modifier", "depth_size", "stale", "replaced_image", "finalize"]) +def test_failed_response_keeps_source_and_cleans_owned_data(source, artifacts, monkeypatch, failure): + result, operator = artifacts + before = {name: set(getattr(bpy.data, name)) for name in ("images", "objects", "meshes", "materials")} + if failure == "modifier": + monkeypatch.setattr(conversion, "set_modifier_input", Mock(side_effect=RuntimeError("Failed modifier"))) + elif failure == "finalize": + monkeypatch.setattr(conversion, "finalize_object_result", Mock(side_effect=RuntimeError("Failed finalize"))) + elif failure == "stale": + operator.source_identity = "replaced" + elif failure == "replaced_image": + operator.image_identity = "replaced" + else: + (result.directory / "depth.json").write_text(json.dumps({"projection": "equirectangular", "image_size": [16, 8], "reference_distance": 3})) + with pytest.raises((RuntimeError, ValueError)): + conversion.create_panorama_from_result(bpy.context, result, operator) + for name, values in before.items(): + assert set(getattr(bpy.data, name)) == values + + +def test_panorama_conversion_undo_restores_empty(source, artifacts): + result, operator = artifacts + name = source.name + image_name = source.data.name + original_space, original_alpha = source.data.colorspace_settings.name, source.data.alpha_mode + bpy.context.preferences.edit.use_global_undo = True + def execute(self, context): + conversion.create_panorama_from_result(context, result, operator) + return {"FINISHED"} + with patch.object(operators.GeneratePanorama, "execute", execute): + bpy.utils.register_class(operators.GeneratePanorama) + try: + bpy.ops.ed.undo_push(message="Before Panorama") + assert bpy.ops.anyimage.generate_panorama("EXEC_DEFAULT") == {"FINISHED"} + assert bpy.data.objects[name].type == "MESH" + assert bpy.ops.ed.undo() == {"FINISHED"} + assert bpy.data.objects[name].type == "EMPTY" + assert (bpy.data.images[image_name].colorspace_settings.name, + bpy.data.images[image_name].alpha_mode) == (original_space, original_alpha) + assert bpy.ops.ed.redo() == {"FINISHED"} + assert bpy.data.objects[name].type == "MESH" + finally: + bpy.utils.unregister_class(operators.GeneratePanorama) diff --git a/tests/anyimage/operators/test_plane_operators.py b/tests/anyimage/operators/test_plane_operators.py index dfeb855..904f9a5 100644 --- a/tests/anyimage/operators/test_plane_operators.py +++ b/tests/anyimage/operators/test_plane_operators.py @@ -1,54 +1,37 @@ from pathlib import Path +from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import Mock, patch from tests.support.blender import BlenderTestCase class PlaneOperatorsTest(BlenderTestCase): - def test_plane_depth_off_accepts_animated_image_without_server(self): - operator = SimpleNamespace( - depth_mode="OFF", - remove_background=False, - shadeless=False, - profile="SURFACE", - mesh_detail=64, - thickness=0.1, - inflation=0.1, - algorithm="POISSON", - ) - source = SimpleNamespace(data=SimpleNamespace(name="Sequence")) - context = SimpleNamespace(object=source, scene=object()) - - with ( - patch.object(self.convert_to_plane, "video_total_frames", return_value=24), - patch.object( - self.convert_to_plane, - "image_user_settings", - return_value=(1, 0, 24, True), - ), - patch.object(self.convert_to_plane, "create_image_material", return_value="Material"), - patch.object(self.convert_to_plane, "create_color_image", return_value="Color"), - patch.object(self.convert_to_plane, "create_plane_object") as build_mesh, - ): - result = self.convert_to_plane.ConvertToPlane.execute(operator, context) - - self.assertEqual(result, {"FINISHED"}) - build_mesh.assert_called_once() + def test_plane_rejects_animated_images_before_creating_resources(self): + for kind in ("MOVIE", "SEQUENCE"): + for duration in (1, 24): + source = SimpleNamespace(data=SimpleNamespace(source=kind, frame_duration=duration)) + operator = SimpleNamespace(report=Mock()) + with patch.object(self.convert_to_plane, "material_color_image") as prepare: + result = self.convert_to_plane.ConvertToPlane.execute( + operator, SimpleNamespace(object=source)) + self.assertEqual(result, {"CANCELLED"}) + prepare.assert_not_called() + operator.report.assert_called_once_with({"ERROR"}, "Material conversion supports static images only") def test_depth_plane_rejects_animation_and_requests_ai_setup(self): operator = SimpleNamespace(mesh_detail=6, report=Mock()) context = SimpleNamespace(object=SimpleNamespace(data=object())) with ( - patch.object(self.convert_to_plane, "video_total_frames", return_value=2), + patch.object(self.convert_to_plane, "require_static_color_image", side_effect=ValueError("Material conversion supports static images only")), patch.object(self.convert_to_plane, "invoke_ai_setup_if_needed") as setup, ): self.assertEqual(self.convert_to_plane.ConvertToDepthPlane.execute(operator, context), {"CANCELLED"}) setup.assert_not_called() with ( - patch.object(self.convert_to_plane, "video_total_frames", return_value=1), + patch.object(self.convert_to_plane, "require_static_color_image"), patch.object(self.convert_to_plane, "invoke_ai_setup_if_needed", return_value={"FINISHED"}) as setup, - patch.object(self.convert_to_plane, "prepare_image_input") as prepare, + patch.object(self.convert_to_plane, "prepare_material_color_input") as prepare, ): self.assertEqual(self.convert_to_plane.ConvertToDepthPlane.execute(operator, context), {"FINISHED"}) setup.assert_called_once() @@ -58,14 +41,15 @@ class PlaneOperatorsTest(BlenderTestCase): def test_depth_and_relief_start_jobs_with_their_plane_type(self): for plane_type, operator_class in (("DEPTH", self.convert_to_plane.ConvertToDepthPlane), ("RELIEF", self.convert_to_plane.ConvertToReliefPlane)): module = self.convert_to_plane - source = SimpleNamespace(name="Source", data=object()) + source = SimpleNamespace(name="Source", as_pointer=lambda: 1, + data=SimpleNamespace(as_pointer=lambda: 2)) context = SimpleNamespace(object=source, scene=object()) operator = SimpleNamespace(mesh_detail=4, report=Mock()) generate = Mock(return_value={"FINISHED"}) with ( - patch.object(module, "video_total_frames", return_value=1), patch.object(module, "invoke_ai_setup_if_needed", return_value=None), - patch.object(module, "prepare_image_input", return_value=(Path("input.png"), True)), + patch.object(module, "prepare_material_color_input", return_value=Path("input.png")), + patch.object(module, "material_analysis_input", return_value=Path("input.png")), patch.object(module, "depth_plane_settings", return_value=("moge", 5)), patch.object(module, "configured_max_ai_input_size", return_value=1024), patch.object(module.bpy, "ops", SimpleNamespace(anyimage=SimpleNamespace(generate_depth_plane=generate)), create=True), @@ -94,15 +78,15 @@ class PlaneOperatorsTest(BlenderTestCase): directory=Path("result"), file=lambda key: files[key], ) - operator = SimpleNamespace(source_object_name="Source", mesh_detail=6, plane_type=plane_type) + operator = SimpleNamespace(source_object_name="Source", color_path="color.png", mesh_detail=6, plane_type=plane_type) context = SimpleNamespace(scene=object()) ops = SimpleNamespace( ed=SimpleNamespace(undo_push=Mock(return_value={"FINISHED"})) ) with ( - patch.object(module, "require_image_empty", return_value=source), - patch.object(module, "load_color_result_image", return_value=color_image), + patch.object(module, "require_conversion_source", return_value=source), + patch.object(module, "material_color_image", return_value=nullcontext(color_image)), patch.object(module, "load_depth_metadata", return_value=metadata), patch.object(module, "load_depth_result_image", return_value=depth_image) as load_depth, patch.object( @@ -165,8 +149,8 @@ class PlaneOperatorsTest(BlenderTestCase): ) with ( - patch.object(module, "require_image_empty", return_value=source), - patch.object(module, "load_color_result_image", return_value=color_image), + patch.object(module, "require_conversion_source", return_value=source), + patch.object(module, "material_color_image", return_value=nullcontext(color_image)), patch.object( module, "load_depth_metadata", @@ -186,13 +170,13 @@ class PlaneOperatorsTest(BlenderTestCase): module.create_depth_plane_from_result( SimpleNamespace(scene=object()), result, - SimpleNamespace(source_object_name="Source", mesh_detail=6, plane_type="DEPTH"), + SimpleNamespace(source_object_name="Source", color_path="color.png", mesh_detail=6, plane_type="DEPTH"), ) remove_material.assert_called_once_with(material, do_unlink=True) self.assertEqual( [entry.args[0] for entry in remove_image.call_args_list], - [color_image, depth_image, normal_image], + [depth_image, normal_image], ) self.assertTrue( all( diff --git a/tests/anyimage/server/test_background_alpha.py b/tests/anyimage/server/test_background_alpha.py new file mode 100644 index 0000000..edf9586 --- /dev/null +++ b/tests/anyimage/server/test_background_alpha.py @@ -0,0 +1,36 @@ +from unittest.mock import patch + +import numpy as np +from PIL import Image +import pytest + +from server.jobs import remove_background +from server.models import onnx_ben2 +from tests.anyimage.server.support import FakeJobContext + + +@pytest.mark.parametrize("failure", ["cancel", "prediction", "sequence", "output_kind"]) +def test_alpha_job_failure_does_not_write_result(tmp_path, failure): + source = tmp_path / "input.png" + Image.new("RGB", (2, 2)).save(source) + context = FakeJobContext(tmp_path / "job", "remove-background") + parameters = {"input": str(source), "model": "BEN2_BASE", "device": "cpu", "output_kind": "alpha"} + if failure == "sequence": + frames = tmp_path / "frames" + frames.mkdir() + for i in (1, 2): + Image.new("RGB", (2, 2)).save(frames / f"{i}.png") + parameters["input"] = str(frames) + if failure == "output_kind": + parameters["output_kind"] = "invalid" + + def infer(*args, **kwargs): + if failure == "prediction": + raise RuntimeError("Prediction failed") + context.check_cancelled = lambda: (_ for _ in ()).throw(RuntimeError("Cancelled")) + return np.ones((2, 2), np.float32) + + with patch.object(onnx_ben2, "create_session", return_value=object()), patch.object(onnx_ben2, "infer_alpha", side_effect=infer): + with pytest.raises((RuntimeError, ValueError)): + remove_background.run(context, parameters) + assert not (context.directory / "alpha.npy").exists() diff --git a/tests/anyimage/server/test_edge_depth.py b/tests/anyimage/server/test_edge_depth.py index d9fe59a..90bae2b 100644 --- a/tests/anyimage/server/test_edge_depth.py +++ b/tests/anyimage/server/test_edge_depth.py @@ -204,7 +204,7 @@ def test_artifacts_use_corrected_depth_and_original_normals( check_cancelled=lambda: None, ) with ( - patch.object(artifacts.moge2, "infer", return_value=[frame]), + patch.object(artifacts.moge, "infer", return_value=[frame]), patch.object( artifacts, "write_depth_texture", return_value=tmp_path / "depth.exr", @@ -217,7 +217,7 @@ def test_artifacts_use_corrected_depth_and_original_normals( ) as normal, patch.object(artifacts, "extend_edge_depth", wraps=extend_edge_depth) as repair, ): - artifacts.generate_moge2_artifacts( + artifacts.generate_moge_artifacts( context, {}, path, generate_depth=generate_depth, normal_mode=normal_mode, ) if normal_mode != "NONE": diff --git a/tests/anyimage/server/test_geometry_artifacts.py b/tests/anyimage/server/test_geometry_artifacts.py index 1fa9757..e46db3e 100644 --- a/tests/anyimage/server/test_geometry_artifacts.py +++ b/tests/anyimage/server/test_geometry_artifacts.py @@ -19,7 +19,7 @@ class GeometryArtifactsTest(ServerTestCase): validity=validity.copy(), intrinsics=np.eye(3, dtype=np.float32), ) output = self.storage_root / "depth.exr" - with patch.object(depth, "_write_float_exr") as write: + with patch.object(depth, "write_float_exr") as write: depth.write_depth_texture(frame, output, alpha=alpha) packed = write.call_args.args[0] np.testing.assert_allclose(packed[..., 3], [[0.6, 0, 0, 1]]) @@ -246,10 +246,10 @@ class GeometryArtifactsTest(ServerTestCase): texture = importlib.import_module("server.geometry.depth_texture") with ( - patch.object(artifacts.moge2, "infer", return_value=[frame]) as infer, - patch.object(texture, "_write_float_exr", wraps=texture._write_float_exr) as write, + patch.object(artifacts.moge, "infer", return_value=[frame]) as infer, + patch.object(texture, "write_float_exr", wraps=texture.write_float_exr) as write, ): - result = artifacts.generate_moge2_artifacts( + result = artifacts.generate_moge_artifacts( context, {"max_input_size": 2048}, input_path, @@ -284,7 +284,7 @@ class GeometryArtifactsTest(ServerTestCase): normal = context.directory / "object-normal.png" with patch.object( job, - "generate_moge2_artifacts", + "generate_moge_artifacts", return_value={ "depth": depth_texture, "depth_metadata": metadata, diff --git a/tests/anyimage/server/test_input_validation.py b/tests/anyimage/server/test_input_validation.py new file mode 100644 index 0000000..832e6c7 --- /dev/null +++ b/tests/anyimage/server/test_input_validation.py @@ -0,0 +1,93 @@ +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from PIL import Image + +from tests.anyimage.server.support import load_server_module + + +@pytest.fixture +def media(): + return load_server_module("media.input") + + +@pytest.mark.parametrize("suffix", [".PNG", ".MP4", ".webp", ".mts"]) +def test_supported_format_is_case_insensitive(media, tmp_path, suffix): + path = tmp_path / f"input{suffix}" + path.touch() + assert media.validate_input_path(path) == path + + +@pytest.mark.parametrize("suffix", [".tiff", ""]) +def test_frontend_and_backend_share_format_reason(media, tmp_path, suffix): + path = tmp_path / f"input{suffix}" + path.touch() + with pytest.raises(RuntimeError) as preflight: + media.validate_input_path(path) + with pytest.raises(RuntimeError) as backend: + media.prepare_input(path, 16, tmp_path) + assert str(preflight.value) == str(backend.value) + assert f"Unsupported input format: {suffix or '(no extension)'}" in str(backend.value) + assert "Supported images: .jpeg, .jpg, .png, .webp." in str(backend.value) + assert "Supported videos:" in str(backend.value) + + +@pytest.mark.parametrize("suffix,kind", [(".png", "image"), (".mp4", "video")]) +def test_missing_path_precedes_format_validation(media, tmp_path, suffix, kind): + with pytest.raises(RuntimeError, match=f"Source {kind} file does not exist"): + media.validate_input_path(tmp_path / f"missing{suffix}") + + +def test_directory_filtering_and_empty_reason(media, tmp_path): + (tmp_path / "note.txt").touch() + (tmp_path / "frame.exr").touch() + with pytest.raises(RuntimeError, match="No supported image frames found"): + media.validate_input_path(tmp_path) + for name in ["b.PNG", "a.jpg"]: + (tmp_path / name).touch() + frames, temporary = media.prepare_input(tmp_path, 1, tmp_path, start_frame=2) + assert frames == [tmp_path / "b.PNG"] + assert temporary is None + + +def test_image_subset_rejects_video(media, tmp_path): + path = tmp_path / "clip.mp4" + path.touch() + with pytest.raises(RuntimeError) as error: + media.validate_input_path(path, images_only=True) + assert "Supported images:" in str(error.value) + assert "Supported videos:" not in str(error.value) + + +def test_image_decode_error_keeps_cause(media, tmp_path): + path = tmp_path / "broken.png" + path.write_bytes(b"broken") + with pytest.raises(RuntimeError, match="Unable to read image") as error: + with media.open_image(path): + pytest.fail("Broken image must not reach processing") + assert isinstance(error.value.__cause__, OSError) + + +def test_processing_error_is_not_relabelled(media, tmp_path): + path = tmp_path / "valid.png" + Image.new("RGB", (2, 2)).save(path) + failure = OSError("inference failed") + with pytest.raises(OSError) as error: + with media.open_image(path): + raise failure + assert error.value is failure + + +def test_video_decode_error_closes_container(media, tmp_path): + failure = ValueError("broken packet") + def frames(): + raise failure + yield + container = SimpleNamespace(streams=SimpleNamespace(video=[object()]), + decode=lambda _stream: frames(), close=Mock()) + with pytest.raises(RuntimeError, match="Unable to read video") as error: + media.extract_video_frames("broken.mp4", tmp_path, 2, + SimpleNamespace(open=lambda _path: container)) + assert error.value.__cause__ is failure + container.close.assert_called_once() diff --git a/tests/anyimage/server/test_job_server.py b/tests/anyimage/server/test_job_server.py index 9a0ed3b..8617a65 100644 --- a/tests/anyimage/server/test_job_server.py +++ b/tests/anyimage/server/test_job_server.py @@ -165,6 +165,7 @@ class JobServerTest(ServerTestCase): "remove-background", "upscale-image", "generate-depth-plane-geometry", + "generate-panorama-geometry", "generate-cutout-artifacts", }, ) diff --git a/tests/anyimage/server/test_media_input.py b/tests/anyimage/server/test_media_input.py index 71932ed..2731e73 100644 --- a/tests/anyimage/server/test_media_input.py +++ b/tests/anyimage/server/test_media_input.py @@ -103,22 +103,6 @@ class MediaInputTest(unittest.TestCase): self.assertIsNone(temporary_directory) - def test_copy_color_images_numbers_a_still_image(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - source = root / "source.png" - source.write_bytes(b"frame") - output = root / "result" - output.mkdir() - - self.media.copy_color_images([source], output) - - self.assertEqual( - (output / "image.0001.png").read_bytes(), - b"frame", - ) - - def test_extract_video_frames_reads_frames_sequentially(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -179,5 +163,5 @@ class MediaInputTest(unittest.TestCase): input_path = Path(directory) / "input.txt" input_path.touch() - with self.assertRaisesRegex(RuntimeError, "Unsupported input type"): + with self.assertRaisesRegex(RuntimeError, "Unsupported input format: .txt"): self.media.prepare_input(input_path, 16, directory) diff --git a/tests/anyimage/server/test_model_ben2_onnx.py b/tests/anyimage/server/test_model_ben2_onnx.py index 9c5a078..6e80e5d 100644 --- a/tests/anyimage/server/test_model_ben2_onnx.py +++ b/tests/anyimage/server/test_model_ben2_onnx.py @@ -36,13 +36,13 @@ class Ben2OnnxTest(unittest.TestCase): names, feeds ), ) as run: - result = onnx_ben2.infer( + result = onnx_ben2.infer_alpha( session, Image.new("RGB", (40, 20)), release_memory=False, ) - self.assertEqual(result.size, (40, 20)) + self.assertEqual(result.shape, (20, 40)) self.assertFalse(run.call_args.kwargs["release_memory"]) def test_preprocess_matches_model_layout(self): @@ -57,13 +57,18 @@ class Ben2OnnxTest(unittest.TestCase): session = FakeSession() image = Image.new("RGB", (40, 20), (10, 20, 30)) - result = onnx_ben2.infer(session, image) + result = onnx_ben2.infer_alpha(session, image) - self.assertEqual(result.mode, "RGBA") - self.assertEqual(result.size, image.size) + self.assertEqual(result.dtype, np.float32) + self.assertEqual(result.shape, (20, 40)) self.assertEqual(session.feeds["input"].shape, (1, 3, 1024, 1024)) - alpha = np.asarray(result.getchannel("A")) - self.assertLess(int(alpha[:, 0].max()), int(alpha[:, -1].min())) + self.assertLess(result[:, 0].max(), result[:, -1].min()) + self.assertTrue(np.any(np.abs(result * 255 - np.rint(result * 255)) > 0.01)) + + def test_infer_rejects_nonfinite_mask(self): + with patch.object(onnx_ben2, "run_session", return_value=[np.full((1, 1, 2, 2), np.nan)]): + with self.assertRaisesRegex(RuntimeError, "invalid alpha"): + onnx_ben2.infer_alpha(FakeSession(), Image.new("RGB", (2, 2))) if __name__ == "__main__": diff --git a/tests/anyimage/server/test_model_geometry.py b/tests/anyimage/server/test_model_geometry.py index 7b986a9..acca780 100644 --- a/tests/anyimage/server/test_model_geometry.py +++ b/tests/anyimage/server/test_model_geometry.py @@ -2,7 +2,7 @@ from unittest.mock import patch import numpy as np -from anyimage.server.models import moge2 as moge2_model +from anyimage.server.models import moge as moge2_model from anyimage.server.models.geometry import GeometryFrame diff --git a/tests/anyimage/server/test_model_inference.py b/tests/anyimage/server/test_model_inference.py index b00bcf4..1173dc1 100644 --- a/tests/anyimage/server/test_model_inference.py +++ b/tests/anyimage/server/test_model_inference.py @@ -12,7 +12,7 @@ from tests.anyimage.server.support import load_server_module, FakeModels class ModelInferenceTest(unittest.TestCase): @classmethod def setUpClass(cls): - cls.moge2_model = load_server_module("models.moge2") + cls.moge2_model = load_server_module("models.moge") cls.upscale_model = load_server_module("models.upscale") diff --git a/tests/anyimage/server/test_model_manager.py b/tests/anyimage/server/test_model_manager.py index b85f038..fedb4f0 100644 --- a/tests/anyimage/server/test_model_manager.py +++ b/tests/anyimage/server/test_model_manager.py @@ -83,7 +83,7 @@ class ModelManagerTest(ServerTestCase): fake_moge = ModuleType("server.models.onnx_moge2") fake_moge.create_session = lambda _model_dir, _device: new_moge with patch.dict(sys.modules, {"server.models.onnx_moge2": fake_moge}): - self.assertIs(cache.get_moge2("new-moge", "cuda"), new_moge) + self.assertIs(cache.get_moge("new-moge", "cuda"), new_moge) self.assertIs(cache.ben2_model, ben2) self.assertIs(cache.upscale_model, upscale) @@ -167,10 +167,10 @@ class ModelManagerTest(ServerTestCase): "release_ben2", ) - first_moge2 = cache.get_moge2("moge2", "cuda") + first_moge2 = cache.get_moge("moge2", "cuda") assert_reused( first_moge2, - lambda: cache.get_moge2("moge2", "cuda"), + lambda: cache.get_moge("moge2", "cuda"), "release_geometry", ) diff --git a/tests/anyimage/server/test_model_moge3_onnx.py b/tests/anyimage/server/test_model_moge3_onnx.py new file mode 100644 index 0000000..a076a70 --- /dev/null +++ b/tests/anyimage/server/test_model_moge3_onnx.py @@ -0,0 +1,139 @@ +from dataclasses import replace +from unittest.mock import patch + +import numpy as np +import pytest + +from server.models import moge, onnx_moge3 +from server import model_catalog, model_manager + + +def test_sparse_neighbors_and_pooling_follow_voxel_coordinates(): + coord = np.zeros((1, 16, 32, 3), dtype=np.float32) + inputs = onnx_moge3.build_sparse_inputs(coord) + neighbors = inputs["neighbor_0"] + assert np.array_equal(neighbors[:, 13], np.arange(512)) + assert neighbors[0, 10] == 512 # Missing pixel at x=-1. + assert neighbors[0, 16] == 1 + for level in range(4): + children = inputs[f"pool_{level}"] + parent = inputs[f"up_{level}"] + for row, values in enumerate(children): + valid = values < len(parent) + assert np.all(parent[values[valid]] == row) + shifted = coord.copy() + shifted[..., 2] += 1 + for key, values in inputs.items(): + assert np.array_equal(values, onnx_moge3.build_sparse_inputs(shifted)[key]) + + +def test_inference_runs_three_refinements_and_preserves_image_dimensions(): + calls = [] + coord = np.zeros((1, 16, 16, 3), dtype=np.float32) + backbone = object() + refiner = object() + + def run(session, names, inputs, *, release_memory): + calls.append((session, inputs.copy(), release_memory)) + if session is backbone: + return [ + coord, + np.zeros((1, 1026, 1, 1), np.float32), + np.ones((1, 3, 16, 16), np.float32), + np.ones((1, 1, 16, 16), np.float32), + np.zeros((1, 1), np.float32), + ] + assert "neighbor_4" in inputs + return [np.full((1, 16, 16), 0.1, np.float32)] + + with ( + patch.object(onnx_moge3, "run_session", side_effect=run), + patch.object( + onnx_moge3, + "postprocess", + side_effect=lambda value, **kwargs: (value, kwargs), + ), + ): + output, options = moge.infer_image( + onnx_moge3.Moge3Session(backbone, refiner), + np.zeros((7, 11, 3), np.float32), + 9, + fov_x=90, + ) + assert len(calls) == 4 + assert calls[0][1]["num_tokens"] == 3600 + assert [call[2] for call in calls[1:]] == [False, False, True] + assert output["points"].shape == (1, 7, 11, 3) + assert np.allclose(output["points"][..., 2], np.exp(0.3)) + assert options["fov_x"] == 90 + + +def test_bilinear_resize_preserves_edges_and_pixel_centers(): + image = np.array([[0, 2], [4, 6]], np.float32) + assert np.array_equal(onnx_moge3._resize(image, 2, 2), image) + assert onnx_moge3._resize(image, 1, 1)[0, 0] == 3 + + +def test_moge3_requires_all_verified_model_files(tmp_path): + model = replace( + model_catalog.MOGE3_VITL, + r2_files=( + ( + "backbone.onnx", + 1, + "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb", + ), + ( + "refiner.onnx", + 1, + "3e23e8160039594a33894f6564e1b1348bbd7a0088d42c4acb73eeaed59c009d", + ), + ), + ) + manager = model_manager.ModelManager(tmp_path) + directory = manager.directory("MOGE3_VITL") + directory.mkdir() + with patch.object( + model_manager.model_catalog, "get_downloadable_model", return_value=model + ): + (directory / "backbone.onnx").write_bytes(b"a") + assert not manager.ready("MOGE3_VITL") + (directory / "refiner.onnx").write_bytes(b"b") + assert manager.ready("MOGE3_VITL") + (directory / "refiner.onnx").write_bytes(b"c") + assert not manager.ready("MOGE3_VITL") + + +def test_cache_reuses_moge3_and_releases_it_when_switching(): + cache = model_manager.ModelSessionCache() + session = onnx_moge3.Moge3Session(object(), object()) + with patch.object(onnx_moge3, "create_session", return_value=session) as load: + assert cache.get_moge("moge-3-vitl-onnx", "cpu") is session + assert cache.get_moge("moge-3-vitl-onnx", "cpu") is session + load.assert_called_once() + from server.models import onnx_moge2 + + with patch.object(onnx_moge2, "create_session", return_value=object()): + assert cache.get_moge("moge-2-vits-normal-onnx", "cpu") is not session + cache.close() + assert cache.moge_model is None + + +def test_moge3_catalog_uses_onnx_assets_and_geometry_family(): + model = model_catalog.get_downloadable_model("MOGE3_VITL") + assert model.ready_patterns == ("backbone.onnx", "refiner.onnx") + assert model_catalog.model_record(model)["family"] == "moge" + assert model.huggingface_repository == "" + + +def test_cache_recovers_after_moge3_session_load_failure(): + cache = model_manager.ModelSessionCache() + session = onnx_moge3.Moge3Session(object(), object()) + with patch.object( + onnx_moge3, "create_session", side_effect=[RuntimeError("load failed"), session] + ): + with pytest.raises(RuntimeError, match="load failed"): + cache.get_moge("moge-3-vitl-onnx", "cpu") + assert cache.moge_model is None + assert cache.moge_key is None + assert cache.get_moge("moge-3-vitl-onnx", "cpu") is session diff --git a/tests/anyimage/server/test_panorama.py b/tests/anyimage/server/test_panorama.py new file mode 100644 index 0000000..8fba1d5 --- /dev/null +++ b/tests/anyimage/server/test_panorama.py @@ -0,0 +1,133 @@ +"""Analytic projection, production inference and artifact lifecycle checks.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest +from PIL import Image + +from tests.anyimage.server.support import FakeJobContext, load_server_module + +geometry = load_server_module("geometry.panorama") +onnx = load_server_module("models.onnx_moge2") +job = load_server_module("jobs.panorama") + + +def test_known_fov_recovers_shift_metric_and_excludes_invisible_points(): + rays = geometry.view_rays(64) + depth = 2 + 0.3 * rays[..., 0] + points = rays * depth[..., None] + points[..., 2] -= 0.7 + visible = np.ones((64, 64), bool) + visible[:8] = False + points[:8] = np.nan + raw = {"points": points[None], "normal": np.zeros_like(points)[None], + "mask": np.ones((1, 64, 64)), "metric_scale": [1.5]} + prediction = onnx.postprocess(raw, fov_x=90, source_valid=visible) + np.testing.assert_allclose(prediction["depth"][visible], depth[visible] * 1.5, rtol=1e-5) + assert not prediction["mask"][:8].any() + assert np.isfinite(prediction["points"]).all() + + +def test_periodic_fusion_removes_view_scale_offsets(): + bases = geometry.camera_bases() + distances = [] + for scale, basis in zip(np.exp(np.linspace(-0.4, 0.4, 12)), bases): + rays = geometry.view_rays(128) @ basis + rays /= np.linalg.norm(rays, axis=-1, keepdims=True) + distances.append((3 + 0.4 * rays[..., 0] + 0.2 * rays[..., 2]) * scale) + merged, valid = geometry.merge_distances(distances, np.ones((12, 128, 128), bool), bases, 64, 32) + rays = geometry.panorama_directions(64, 32) + expected = 3 + 0.4 * rays[..., 0] + 0.2 * rays[..., 2] + merged *= np.median(expected / merged) + assert valid.all() + assert np.mean(abs(merged / expected - 1)) < 0.002 + assert np.max(abs(merged[:, [0, -1]] / expected[:, [0, -1]] - 1)) < 0.005 + sampled = geometry.sample_panorama(expected, np.array(((1, 1e-8, 0), (1, -1e-8, 0)))) + np.testing.assert_allclose(sampled[0], sampled[1], atol=1e-6) + + +@pytest.fixture +def panorama_job(tmp_path, monkeypatch): + source = tmp_path / "source.png" + context = FakeJobContext(tmp_path / "result", "generate-panorama-geometry", + SimpleNamespace(directory=lambda _: tmp_path, get_moge=lambda *_: object())) + calls, pixels = [], [] + + def infer(session, color, level, **options): + calls.append((color, level, options)) + rays = geometry.view_rays(color.shape[0]) + points = rays / np.linalg.norm(rays, axis=-1, keepdims=True) * 3 + return {"points": points, "mask": np.ones(color.shape[:2])} + + monkeypatch.setattr(job, "infer", infer) + write = job.write_float_exr + monkeypatch.setattr(job, "write_float_exr", lambda rgba, path: (pixels.append(rgba.copy()), write(rgba, path))[1]) + parameters = {"input": str(source), "model": "MOGE2", "device": "cpu", "resolution_level": 3, + "max_input_size": 32, "video_frames": 1, "start_frame": 1} + return source, context, parameters, calls, pixels + + +def test_job_outputs_geometry_combines_alpha_once_and_skips_transparent_views(panorama_job): + source, context, parameters, calls, pixels = panorama_job + image = np.full((32, 64, 4), 255, np.uint8) + image[:24, :, 3] = 0 + image[26:28, :, 3] = 128 + Image.fromarray(image).save(source) + result = job.run(context, parameters) + assert 0 < len(calls) < 12 + assert all(level == 3 and opts["fov_x"] == 90 for _, level, opts in calls) + assert all(np.all(color[~opts["source_valid"]] == 0) for color, _, opts in calls) + assert {path.name for path in context.directory.iterdir()} == {"depth.exr", "depth.json"} + alpha = pixels[0][..., 3] + assert pixels[0].shape == image.shape + assert not alpha[:24].any() + assert np.all(alpha <= image[..., 3] / 255 + 1e-7) + assert np.any(np.isclose(alpha, 128 / 255)) + np.testing.assert_allclose(pixels[0][..., 0], pixels[0][..., 2]) + metadata = json.loads((context.directory / result["depth_metadata"]).read_text()) + assert metadata["projection"] == "equirectangular" + assert metadata["image_size"] == [64, 32] + assert metadata["reference_distance"] == pytest.approx(3, rel=1e-5) + + +def test_full_resolution_fusion_preserves_detail_and_periodic_alignment(): + bases = geometry.camera_bases() + distances = [] + def radius(rays): + angle = np.arctan2(rays[..., 1], rays[..., 0]) + return 3 + 0.15 * np.sin(80 * angle) * (1 - rays[..., 2] ** 2) ** 4 + for scale, basis in zip(np.exp(np.linspace(-0.2, 0.2, 12)), bases): + rays = geometry.view_rays(512) @ basis + rays /= np.linalg.norm(rays, axis=-1, keepdims=True) + distances.append(radius(rays) * scale) + merged, valid = geometry.merge_distances(distances, np.ones((12, 512, 512), bool), + bases, 1024, 512) + expected = radius(geometry.panorama_directions(1024, 512)) + merged *= np.median(expected / merged) + assert merged.shape == (512, 1024) and valid.all() + assert np.sqrt(np.mean((merged - expected) ** 2)) < 0.001 + assert np.max(abs(merged[:, [0, -1]] / expected[:, [0, -1]] - 1)) < 0.01 + + +@pytest.mark.parametrize("kind", ["transparent", "invalid_model", "wrong_ratio", "cancel"]) +def test_failed_jobs_leave_no_partial_artifacts(panorama_job, monkeypatch, kind): + source, context, parameters, calls, _ = panorama_job + Image.new("RGBA", (64 if kind != "wrong_ratio" else 32, 32), + (255, 255, 255, 0 if kind == "transparent" else 255)).save(source) + if kind == "invalid_model": + monkeypatch.setattr(job, "infer", lambda *a, **k: {"points": np.ones((32, 32, 3)), "mask": np.zeros((32, 32))}) + if kind == "cancel": + count = [0] + def cancel(): + count[0] += 1 + if count[0] == 6: + raise RuntimeError("Cancelled") + context.check_cancelled = cancel + with pytest.raises((ValueError, RuntimeError)): + job.run(context, parameters) + assert not list(context.directory.glob("depth.*")) + assert not (context.directory / "image.0001.png").exists() + if kind == "transparent": + assert not calls diff --git a/tests/anyimage/server/test_server_jobs.py b/tests/anyimage/server/test_server_jobs.py index 253d29f..a759ee9 100644 --- a/tests/anyimage/server/test_server_jobs.py +++ b/tests/anyimage/server/test_server_jobs.py @@ -54,10 +54,9 @@ class ServerJobsTest(unittest.TestCase): "input_frames", return_value=nullcontext([input_path]), ), - patch.object(self.depth_plane_job, "copy_color_images"), patch.object( self.depth_plane_job, - "generate_moge2_artifacts", + "generate_moge_artifacts", return_value={ "depth": depth_texture, "depth_metadata": metadata, @@ -85,100 +84,23 @@ class ServerJobsTest(unittest.TestCase): def test_ben2_background_removal_writes_fast_transparent_png(self): - inference_options = [] - save_options = [] - - class FakeImage: - size = (1, 1) - - def __enter__(self): - return self - - def __exit__(self, *_args): - return None - - def convert(self, mode): - self.mode = mode - return self - - def getchannel(self, _name): - return self - - def putalpha(self, _alpha): - return None - - def paste(self, *_args): - return None - - def save(self, path, **options): - save_options.append(options) - Path(path).touch() - - class FakeModel: - @classmethod - def from_pretrained(cls, model_dir): - self.assertEqual(model_dir, "model") - return cls() - - def to(model_self, device): - self.assertEqual(device.type, "cpu") - return model_self - - def eval(self): - return None - - def inference(self, image, **options): - inference_options.append((image, options)) - return FakeImage() - - fake_torch = ModuleType("torch") - fake_torch.cuda = SimpleNamespace( - is_available=lambda: False, - synchronize=lambda: None, - ) - fake_torch.device = lambda device_type: SimpleNamespace(type=device_type) - fake_torch.inference_mode = nullcontext - fake_ben2 = ModuleType("server.models.onnx_ben2") - fake_ben2.create_session = lambda _model_dir, _device: object() - fake_ben2.infer = lambda _model, image, **options: ( - inference_options.append((image, options)) or FakeImage() - ) - fake_pil = ModuleType("PIL") - fake_pil.Image = SimpleNamespace( - open=lambda _path: FakeImage(), - new=lambda *_args, **_kwargs: FakeImage(), - ) - fake_pil.ImageChops = SimpleNamespace( - multiply=lambda _source, _foreground: FakeImage() - ) - - with tempfile.TemporaryDirectory() as directory, patch.dict( - sys.modules, - { - "torch": fake_torch, - "server.models.onnx_ben2": fake_ben2, - "PIL": fake_pil, - }, - ): + from PIL import Image + onnx_ben2 = load_server_module("models.onnx_ben2") + with tempfile.TemporaryDirectory() as directory: root = Path(directory) input_path = root / "input.png" - input_path.touch() + Image.new("RGBA", (2, 1), (255, 64, 32, 128)).save(input_path) context = FakeJobContext(root / "result", "remove-background") - self.background.run( - context, - {"input": str(input_path), "model": "BEN2", "device": "cpu"}, - ) - - self.assertTrue((root / "result" / "image.0001.png").is_file()) - - self.assertEqual( - inference_options[0][1], - {"release_memory": True}, - ) - self.assertEqual( - save_options[0], - {"format": "PNG", "compress_level": 1}, - ) + with ( + patch.object(onnx_ben2, "create_session", return_value=object()), + patch.object(onnx_ben2, "infer_alpha", return_value=np.full((1, 2), 128 / 255, np.float32)) as infer, + patch.object(Image.Image, "save", autospec=True, side_effect=Image.Image.save) as save, + ): + self.background.run(context, {"input": str(input_path), "model": "BEN2", "device": "cpu"}) + with Image.open(root / "result" / "image.0001.png") as image: + self.assertEqual(image.getpixel((0, 0)), (255, 64, 32, 64)) + self.assertEqual(infer.call_args.kwargs, {"release_memory": True}) + self.assertEqual(save.call_args.kwargs, {"format": "PNG", "compress_level": 1}) self.assertIn("Background removed in", context.status["message"]) @@ -190,7 +112,7 @@ class ServerJobsTest(unittest.TestCase): (((255, 0, 0, 0), (0, 255, 0, 128)),), dtype=np.uint8, ) - model_result = Image.new("RGBA", (2, 1), (10, 20, 30, 128)) + model_result = np.full((1, 2), 128 / 255, np.float32) received = [] def infer(_model, image, **_options): @@ -202,7 +124,7 @@ class ServerJobsTest(unittest.TestCase): Image.fromarray(source_pixels, mode="RGBA").save(input_path) with ( patch.object(onnx_ben2, "create_session", return_value=object()), - patch.object(onnx_ben2, "infer", side_effect=infer), + patch.object(onnx_ben2, "infer_alpha", side_effect=infer), ): result, _load_ms, _inference_ms = self.ben2.infer_foreground( input_path, @@ -226,111 +148,48 @@ class ServerJobsTest(unittest.TestCase): self.assertEqual(tuple(result_pixels[0, 1]), (0, 255, 0, 64)) self.assertEqual( tuple(np.asarray(raw_result.convert("RGBA"))[0, 0]), - (10, 20, 30, 128), + (0, 0, 0, 128), ) def test_ben2_background_removal_processes_each_frame(self): - inference_options = [] - - class FakeImage: - size = (1, 1) - - def __enter__(self): - return self - - def __exit__(self, *_args): - return None - - def convert(self, mode): - self.mode = mode - return self - - def getchannel(self, _name): - return self - - def putalpha(self, _alpha): - return None - - def paste(self, *_args): - return None - - def save(self, path, **options): - Path(path).touch() - - class FakeModel: - @classmethod - def from_pretrained(cls, model_dir): - return cls() - - def to(model_self, device): - return model_self - - def eval(self): - return None - - def inference(self, image, **options): - inference_options.append((image, options)) - return FakeImage() - - fake_torch = ModuleType("torch") - fake_torch.cuda = SimpleNamespace( - is_available=lambda: False, - synchronize=lambda: None, - ) - fake_torch.device = lambda device_type: SimpleNamespace(type=device_type) - fake_torch.inference_mode = nullcontext - fake_ben2 = ModuleType("server.models.onnx_ben2") - fake_ben2.create_session = lambda _model_dir, _device: object() - fake_ben2.infer = lambda _model, image, **options: ( - inference_options.append((image, options)) or FakeImage() - ) - fake_pil = ModuleType("PIL") - fake_pil.Image = SimpleNamespace( - open=lambda _path: FakeImage(), - new=lambda *_args, **_kwargs: FakeImage(), - ) - fake_pil.ImageChops = SimpleNamespace( - multiply=lambda _source, _foreground: FakeImage() - ) - - with tempfile.TemporaryDirectory() as directory, patch.dict( - sys.modules, - { - "torch": fake_torch, - "server.models.onnx_ben2": fake_ben2, - "PIL": fake_pil, - }, - ): + from PIL import Image + onnx_ben2 = load_server_module("models.onnx_ben2") + with tempfile.TemporaryDirectory() as directory: root = Path(directory) - input_directory = root / "frames" - input_directory.mkdir() - (input_directory / "frame-0001.jpg").touch() - (input_directory / "frame-0002.jpg").touch() + frames = root / "frames" + frames.mkdir() + for index in (1, 2): + Image.new("RGB", (2, 1), (index * 50, 0, 0)).save(frames / f"frame.{index:04d}.png") context = FakeJobContext(root / "result", "remove-background") - self.background.run( - context, - { - "input": str(input_directory), - "model": "BEN2", - "device": "cpu", - "video_frames": 8, - }, - ) + with ( + patch.object(onnx_ben2, "create_session", return_value=object()), + patch.object(onnx_ben2, "infer_alpha", return_value=np.ones((1, 2), np.float32)) as infer, + ): + self.background.run(context, {"input": str(frames), "model": "BEN2", "device": "cpu", "video_frames": 8}) + self.assertEqual([call.kwargs["release_memory"] for call in infer.call_args_list], [False, True]) + for index in (1, 2): + with Image.open(context.directory / f"image.{index:04d}.png") as image: + self.assertEqual(image.getpixel((0, 0)), (index * 50, 0, 0, 255)) - result_directory = context.directory - self.assertTrue( - (result_directory / "image.0001.png").is_file() - ) - self.assertTrue( - (result_directory / "image.0002.png").is_file() - ) - self.assertEqual(len(inference_options), 2) - self.assertEqual( - [options["release_memory"] for _image, options in inference_options], - [False, True], - ) + def test_background_alpha_job_preserves_float_precision(self): + from PIL import Image + onnx_ben2 = load_server_module("models.onnx_ben2") + alpha = np.array([[0.123456, 0.987654]], dtype=np.float32) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "input.png" + Image.new("RGB", (2, 1)).save(path) + context = FakeJobContext(root / "result", "remove-background") + with ( + patch.object(onnx_ben2, "create_session", return_value=object()), + patch.object(onnx_ben2, "infer_alpha", return_value=alpha), + ): + result = self.background.run(context, {"input": str(path), "model": "BEN2", "device": "cpu", "output_kind": "alpha"}) + self.assertEqual(result, {"alpha": "alpha.npy"}) + np.testing.assert_array_equal(np.load(context.directory / result["alpha"], allow_pickle=False), alpha) + self.assertFalse(list(context.directory.glob("*.png"))) def test_upscale_job_saves_importable_upscale_sequence(self): diff --git a/tests/anyimage/test_addon_registration.py b/tests/anyimage/test_addon_registration.py index b80198b..76c922d 100644 --- a/tests/anyimage/test_addon_registration.py +++ b/tests/anyimage/test_addon_registration.py @@ -18,6 +18,16 @@ class AddonRegistrationTest(BlenderTestCase): def test_semantic_mesh_operators_are_registered_without_dialog_options(self): + from anyimage.operators.convert_to_panorama import ConvertToPanorama, GeneratePanorama + + self.assertIn(ConvertToPanorama, self.anyimage.CLASSES) + self.assertIn(GeneratePanorama, self.anyimage.CLASSES) + for generator in (GeneratePanorama, self.convert_to_plane.GenerateDepthPlane): + self.assertIn(generator, self.anyimage.CLASSES) + for name in ("source_object_name", "source_identity", "image_identity"): + self.assertIn(name, generator.__annotations__) + self.assertIn("HIDDEN", generator.__annotations__[name]["options"]) + self.assertLess(self.anyimage.CLASSES.index(GeneratePanorama), self.anyimage.CLASSES.index(ConvertToPanorama)) operator = self.convert_to_plane.ConvertToPlane self.assertIn(operator, self.anyimage.CLASSES) self.assertIn(self.convert_to_plane.ConvertToDepthPlane, self.anyimage.CLASSES) diff --git a/tests/anyimage/test_image_menu.py b/tests/anyimage/test_image_menu.py index 309fdce..70a3896 100644 --- a/tests/anyimage/test_image_menu.py +++ b/tests/anyimage/test_image_menu.py @@ -46,6 +46,7 @@ class ImageMenuTest(BlenderTestCase): ("anyimage.convert_to_plane", True), ("anyimage.convert_to_depth_plane", True), ("anyimage.convert_to_relief_plane", True), + ("anyimage.convert_to_panorama", True), ("separator", True), ("anyimage.remove_image_background", True), ("anyimage.upscale_image", True), diff --git a/tests/anyimage/test_preferences.py b/tests/anyimage/test_preferences.py index 066e898..238e92d 100644 --- a/tests/anyimage/test_preferences.py +++ b/tests/anyimage/test_preferences.py @@ -114,7 +114,7 @@ class PreferencesTest(BlenderTestCase): def test_geometry_model_choices_match_moge_catalog(self): properties = self.anyimage.properties - expected = tuple(properties.shared_model_catalog.MOGE2_MODELS) + expected = tuple(properties.shared_model_catalog.MOGE_MODELS) geometry = properties.geometry_model_enum_items(None, None) self.assertEqual(tuple(item[0] for item in geometry), expected) @@ -222,7 +222,7 @@ class PreferencesTest(BlenderTestCase): "ready": False, } for key, family in ( - ("MOGE2_VITS_NORMAL", "moge2"), + ("MOGE2_VITS_NORMAL", "moge"), ("BEN2_BASE", "ben2"), ("REALESRGAN_GENERAL_WDN_X4V3", "upscale"), ("SAFMN_REAL_X4", "upscale"), @@ -418,8 +418,8 @@ class PreferencesTest(BlenderTestCase): with ( patch.object(self.convert_to_plane, "require_environment"), patch.object(self.convert_to_plane, "require_model"), + patch.object(self.convert_to_plane, "require_input_path", return_value=Path("input.png")), patch.object(self.convert_to_plane, "production_device", return_value="CPU"), - patch.object(self.convert_to_plane, "source_start_frame", return_value=1), ): parameters = operator.request(context) diff --git a/tests/support/color_image.py b/tests/support/color_image.py new file mode 100644 index 0000000..d5f08a6 --- /dev/null +++ b/tests/support/color_image.py @@ -0,0 +1,11 @@ +"""Prepare a standalone color image for material and geometry tests.""" + +from anyimage.common.color_image import prepare_material_color_input, load_material_color_image, cleanup_material_color_input + + +def color_image(source, bounds=None, rgba=None): + path = prepare_material_color_input(source, bounds, rgba) + try: + return load_material_color_image(path) + finally: + cleanup_material_color_input(path) diff --git a/tests/support/hdr_image.py b/tests/support/hdr_image.py new file mode 100644 index 0000000..01e5385 --- /dev/null +++ b/tests/support/hdr_image.py @@ -0,0 +1,15 @@ +import bpy +import numpy as np +import pytest + + +@pytest.fixture +def hdr_texture(texture): + image = bpy.data.images.new("HDR source", width=4, height=3, alpha=True, float_buffer=True) + image.alpha_mode = "PREMUL" + rgba = np.tile([4.0, 2.0, -0.25, 0.5], (3, 4, 1)).astype(np.float32) + rgba[0, 0] = [9, 3, -0.5, 0] + rgba[2, 3] = [25, 12, 6, 1] + image.pixels.foreach_set(np.flipud(rgba).ravel()) + texture.image = image + return texture diff --git a/tests/support/image_objects.py b/tests/support/image_objects.py index ebfe6b8..39cd2d7 100644 --- a/tests/support/image_objects.py +++ b/tests/support/image_objects.py @@ -7,6 +7,7 @@ from anyimage.common import object as object_data from anyimage.common.selection import SelectionMask from anyimage.operators.cutout_tool import geometry as cutout_geometry from anyimage.operators.cutout_tool import object as cutout_object +from tests.support.color_image import color_image as prepared_color_image def _inputs(group): @@ -43,8 +44,8 @@ def _create_cutout_shape( selection_values = np.ones((bottom - top, right - left), dtype=np.float32) selection_mask = SelectionMask(selection_values, bounds) source_rgba = image_data.image_rgba(source.data) - color_image = cutout_object.create_cropped_color_image( - source, + color_image = prepared_color_image( + source.data, selection_mask.bounds, source_rgba, ) diff --git a/tests/tools/nodes/test_depth_relief_planes.py b/tests/tools/nodes/test_depth_relief_planes.py index 35d7558..4ecc82c 100644 --- a/tests/tools/nodes/test_depth_relief_planes.py +++ b/tests/tools/nodes/test_depth_relief_planes.py @@ -197,7 +197,7 @@ def test_surface_interface_and_node_structure(surface): assert inputs["Split Threshold"].subtype == "FACTOR" assert inputs["Split Threshold"].parent == inputs["Smooth"].parent assert inputs["Split Threshold"].parent.name != "Options" - assert inputs["Validity Threshold"].default_value == pytest.approx(0.9) + assert inputs["Validity Threshold"].default_value == pytest.approx(0.1) assert inputs["Validity Threshold"].parent.name == "Options" assert inputs["Validity Threshold"].subtype == "FACTOR" assert inputs["Valid Only"].default_value is True @@ -234,7 +234,7 @@ def test_relief_interface_and_node_structure(relief): @pytest.mark.parametrize("threshold", [0.0, 0.4, 0.9, 1.0]) @pytest.mark.parametrize("pattern", ["valid", "invalid", "partial", "half", "stripe"]) -def test_valid_only_recomputes_per_point_and_preserves_boundary_faces(surface, pattern, threshold): +def test_valid_only_samples_face_centers(surface, pattern, threshold): obj, set_value, _, inputs, _ = surface set_value("Validity Threshold", threshold) depth_image = obj.modifiers[0][inputs["Depth Image"].identifier] @@ -256,17 +256,15 @@ def test_valid_only_recomputes_per_point_and_preserves_boundary_faces(surface, p set_value("Subdivide", subdivisions) set_value("Valid Only", False) base = evaluated(obj) - u = (base[0][:, 0] + 2) / 4 + u = np.array([(base[0][list(face), 0].mean() + 2) / 4 for face in base[1]]) sampled = np.interp(u * 16 - 0.5, np.arange(16), alpha) - expected = [face for face in base[1] if any(sampled[index] >= threshold for index in face)] + expected = [face for face, alpha_value in zip(base[1], sampled) if alpha_value >= threshold] set_value("Valid Only", True) culled = evaluated(obj) assert len(culled[1]) == len(expected) expected_points = {tuple(base[0][index]) for face in expected for index in face} assert {tuple(point) for point in culled[0]} == expected_points assert np.isfinite(culled[2]).all() - if pattern == "half" and subdivisions >= 2 and threshold == 1.0: - assert 0 <= min(point[0] for point in culled[0]) < 0.125 set_value("Valid Only", False) np.testing.assert_allclose(evaluated(obj)[0], base[0]) @@ -293,9 +291,7 @@ def test_validity_graph_matches_reference_domains_and_connections(surface): group = obj.modifiers[0].node_group delete = next(node for node in group.nodes if node.bl_idname == "GeometryNodeDeleteGeometry") assert (delete.domain, delete.mode) == ("FACE", "ALL") - points = origin_socket(delete.inputs["Selection"].links[0].from_socket).node - assert (points.bl_idname, points.data_type, points.domain) == ("GeometryNodeFieldOnDomain", "BOOLEAN", "POINT") - compare = origin_socket(points.inputs["Value"].links[0].from_socket).node + compare = origin_socket(delete.inputs["Selection"].links[0].from_socket).node assert (compare.bl_idname, compare.data_type, compare.operation) == ("FunctionNodeCompare", "FLOAT", "LESS_THAN") assert origin_socket(compare.inputs["B"].links[0].from_socket).name == "Validity Threshold" alpha = origin_socket(compare.inputs["A"].links[0].from_socket) diff --git a/tests/tools/nodes/test_image_depth_panorama.py b/tests/tools/nodes/test_image_depth_panorama.py new file mode 100644 index 0000000..d3f0a7d --- /dev/null +++ b/tests/tools/nodes/test_image_depth_panorama.py @@ -0,0 +1,259 @@ +"""Evaluate panorama topology, validity, UVs and depth discontinuities.""" + +from collections import Counter + +import bpy +import numpy as np +import pytest + +from anyimage.common.object import modifier_input_identifier, set_modifier_input +from tools.nodes.groups.image_depth_panorama import build_image_depth_panorama_group +from tests.support.nodes import validate_node_structure + + +def panorama(depth=3, alpha=1, subdivide=3, reference=3): + bpy.ops.wm.read_factory_settings(use_empty=True) + group = build_image_depth_panorama_group() + assert group.name == "O Image Depth Panorama" + validate_node_structure(group) + assert not any(node.bl_idname == "GeometryNodeGroup" for node in group.nodes) + image = bpy.data.images.new("Radial Depth", width=128, height=64, float_buffer=True) + image.colorspace_settings.name = "Non-Color" + image.alpha_mode = "CHANNEL_PACKED" + rgba = np.ones((64, 128, 4), np.float32) + rgba[..., :3] = np.broadcast_to(depth, (64, 128))[..., None] + rgba[..., 3] = alpha + image.pixels.foreach_set(rgba.ravel()) + obj = bpy.data.objects.new("Panorama", bpy.data.meshes.new("Panorama")) + bpy.context.collection.objects.link(obj) + modifier = obj.modifiers.new("Panorama", "NODES") + modifier.node_group = group + + def set_value(name, value): + set_modifier_input(modifier, modifier_input_identifier(group, name), value) + obj.update_tag(refresh={"DATA"}) + bpy.context.view_layer.update() + + for name, value in (("Depth Image", image), ("Subdivide", subdivide), ("Reference Distance", reference)): + set_value(name, value) + return obj, set_value + + +def evaluated(obj): + result = obj.evaluated_get(bpy.context.evaluated_depsgraph_get()) + mesh = result.to_mesh() + try: + points = np.array([v.co[:] for v in mesh.vertices]).reshape(-1, 3) + faces = [tuple(f.vertices) for f in mesh.polygons] + uv = np.array([v.uv[:] for v in mesh.uv_layers.active.data]) if mesh.uv_layers.active else np.empty((0, 2)) + assert not any(a.name.startswith(("o_depth_", "o_panorama_")) for a in mesh.attributes) + return points, faces, uv + finally: + result.to_mesh_clear() + + +@pytest.mark.parametrize("subdivide", [0, 2, 4]) +def test_quad_sphere_topology_and_radial_endpoints(subdivide): + obj, set_value = panorama(subdivide=subdivide) + inputs = [s for s in obj.modifiers[0].node_group.interface.items_tree if s.item_type == "SOCKET" and s.in_out == "INPUT"] + assert [s.name for s in inputs if s.socket_type == "NodeSocketImage"] == ["Depth Image"] + for scale, radius in ((0, 1), (0.5, 2), (1, 3)): + set_value("Depth Scale", scale) + points, faces, uv = evaluated(obj) + assert len(faces) == 6 * 4 ** subdivide + assert all(len(face) == 4 for face in faces) + np.testing.assert_allclose(np.linalg.norm(points, axis=1), radius, atol=1e-5) + edges = Counter(tuple(sorted((a, b))) for face in faces for a, b in zip(face, (*face[1:], face[0]))) + assert set(edges.values()) == {2} + assert len(points) - len(edges) + len(faces) == 2 + assert np.isfinite(uv).all() + if subdivide > 0: + assert np.max(np.ptp(uv.reshape(-1, 4, 2)[..., 0], axis=1)) <= 0.5 + 1e-5 + + +def test_depth_alpha_culls_faces_without_double_multiplication(): + obj, set_value = panorama(alpha=0.5) + assert not len(evaluated(obj)[0]) + set_value("Valid Only", False) + assert len(evaluated(obj)[1]) == 384 + set_value("Valid Only", True) + set_value("Validity Threshold", 0.49) + assert len(evaluated(obj)[1]) == 384 + + +def test_invalid_depth_does_not_bleed_into_alpha_boundary(): + alpha = np.ones((64, 128)) + alpha[:, 64:] = 0 + depth = np.where(alpha, 3, 10000) + obj, _ = panorama(depth, alpha, subdivide=5) + points, _, _ = evaluated(obj) + np.testing.assert_allclose(np.linalg.norm(points, axis=1), 3, atol=1e-5) + + +def test_linear_depth_reconstructs_flat_floor_without_sampling_rings(): + z = -np.cos((np.arange(64) + 0.5) * np.pi / 64) + depth = np.broadcast_to((2 / np.maximum(-z, 0.1))[:, None], (64, 128)) + obj, _ = panorama(depth, subdivide=6) + points, _, _ = evaluated(obj) + rays = points / np.linalg.norm(points, axis=1, keepdims=True) + floor = rays[:, 2] < -0.5 + error = points[floor, 2] + 2 + assert np.sqrt(np.mean(error ** 2)) < 0.002 + assert np.max(abs(error)) < 0.004 + + +@pytest.mark.parametrize("sky_alpha", [0, 1]) +def test_invalid_distance_removes_faces_between_valid_and_invalid_regions(sky_alpha): + depth = np.full((64, 128), 3.0) + depth[:, 64:] = 1e-7 + alpha = np.ones((64, 128)) + alpha[:, 64:] = sky_alpha + obj, set_value = panorama(depth, alpha, subdivide=4) + points, original_faces, _ = evaluated(obj) + assert np.linalg.norm(points, axis=1).min() > 1 + assert np.linalg.norm(points, axis=1).max() < 3.01 + set_value("Invalid Distance", 250) + assert evaluated(obj)[1] == original_faces + set_value("Valid Only", False) + for background in (100, 250): + set_value("Invalid Distance", background) + points, faces, _ = evaluated(obj) + radii = np.linalg.norm(points, axis=1) + assert np.any(np.isclose(radii, background)) + assert np.any(np.isclose(radii, 3)) + assert len(original_faces) < len(faces) < 6 * 4 ** 4 + assert all(np.all(radii[list(face)] > 10) or np.all(radii[list(face)] < 10) for face in faces) + set_value("Depth Scale", 0) + np.testing.assert_allclose(np.linalg.norm(evaluated(obj)[0], axis=1), 1, atol=1e-5) + + +def test_controls_smoothing_and_material_slot_inheritance(): + obj, set_value = panorama() + group = obj.modifiers[0].node_group + inputs = {s.name: s for s in group.interface.items_tree if s.item_type == "SOCKET" and s.in_out == "INPUT"} + assert inputs["Validity Threshold"].parent.name == "Options" + assert inputs["Invalid Distance"].parent.name == "Options" + assert inputs["Invalid Distance"].default_value == 100 + assert inputs["Invalid Distance"].subtype == "DISTANCE" + assert inputs["Smooth Weight"].parent.name == "Options" + assert inputs["Valid Only"].parent == inputs["Subdivide"].parent + assert inputs["Valid Only"].default_value + assert "Material" not in inputs + obj.data.from_pydata([(0, 0, 0)], [], []) + material = bpy.data.materials.new("Slot Material") + obj.data.materials.append(material) + set_value("Smooth", 0) + original = evaluated(obj) + set_value("Smooth", 2) + smoothed = evaluated(obj) + assert smoothed[1] == original[1] + assert np.max(abs(smoothed[0] - original[0])) > 1e-4 + np.testing.assert_array_equal(smoothed[2], original[2]) + set_value("Smooth Weight", 0) + np.testing.assert_allclose(evaluated(obj)[0], original[0], atol=1e-6) + for name in ("First Slot", "Replacement Slot"): + current = bpy.data.materials.new(name) + obj.data.materials[0] = current + obj.update_tag(refresh={"DATA"}) + bpy.context.view_layer.update() + result = obj.evaluated_get(bpy.context.evaluated_depsgraph_get()) + mesh = result.to_mesh() + try: + assert mesh.materials[0].original == current + assert all(f.material_index == 0 and f.use_smooth for f in mesh.polygons) + finally: + result.to_mesh_clear() + + +def test_material_uv_matches_radial_sampling_and_faces_point_inward(): + obj, _ = panorama(subdivide=4) + result = obj.evaluated_get(bpy.context.evaluated_depsgraph_get()) + mesh = result.to_mesh() + try: + assert all(np.dot(face.normal, face.center) < 0 for face in mesh.polygons) + assert any(loop.uv.x < 0 for loop in mesh.uv_layers.active.data) + for loop in mesh.loops: + ray = np.array(mesh.vertices[loop.vertex_index].co) + ray /= np.linalg.norm(ray) + u, v = mesh.uv_layers.active.data[loop.index].uv + if abs(ray[2]) < 0.999999: + expected_u = np.arctan2(ray[1], ray[0]) / (2 * np.pi) + assert abs((u - expected_u + 0.5) % 1 - 0.5) < 1e-5 + assert abs(v - (1 - np.arccos(np.clip(ray[2], -1, 1)) / np.pi)) < 1e-5 + finally: + result.to_mesh_clear() + + +def test_split_strength_and_distance_scale_invariance(): + depth = np.ones((64, 128)) + depth[:, 64:] = 7 + signatures = [] + for scale in (0.01, 1, 100): + obj, set_value = panorama(depth * scale, reference=3 * scale) + counts = [] + for split in (0, 0.1, 0.5, 1): + set_value("Split Threshold", split) + points, faces, uv = evaluated(obj) + assert np.isfinite(points).all() and np.isfinite(uv).all() + edges = Counter(tuple(sorted((a, b))) for f in faces for a, b in zip(f, (*f[1:], f[0]))) + counts.append(sum(n == 1 for n in edges.values())) + assert counts[0] == 0 and counts[-1] > 0 + assert counts == sorted(counts) + signatures.append(counts) + assert signatures[0] == signatures[1] == signatures[2] + + +def test_continuous_spherical_slope_does_not_split_at_seam_or_poles(): + u, v = np.meshgrid((np.arange(128) + 0.5) / 128, (np.arange(64) + 0.5) / 64) + depth = 3 + 0.3 * np.cos(2 * np.pi * u) * np.sin(np.pi * v) + obj, set_value = panorama(depth) + original = evaluated(obj) + set_value("Split Threshold", 1) + points, faces, uv = evaluated(obj) + np.testing.assert_allclose(points, original[0], atol=1e-6) + assert faces == original[1] + np.testing.assert_allclose(uv, original[2]) + + +def test_split_corners_keep_their_rays_and_own_face_distance(): + from scipy.spatial import cKDTree + + depth = np.ones((64, 128)) + depth[:, 64:] = 7 + obj, set_value = panorama(depth) + original, _, _ = evaluated(obj) + original_radii = np.linalg.norm(original, axis=1) + original_rays = original / original_radii[:, None] + set_value("Split Threshold", 1) + points, _, uv = evaluated(obj) + radii = np.linalg.norm(points, axis=1) + distance, indices = cKDTree(original_rays).query(points / radii[:, None]) + assert distance.max() < 1e-5 + counts = Counter(indices) + uncut = np.array([counts[index] == 1 for index in indices]) + np.testing.assert_allclose(radii[uncut], original_radii[indices[uncut]], atol=1e-5) + assert any(np.ptp(radii[indices == index]) > 4 for index, count in counts.items() if count > 1) + assert np.max(np.ptp(uv.reshape(-1, 4, 2)[..., 0], axis=1)) <= 0.5 + 1e-5 + + +def test_zero_split_bypasses_single_snapshot_strip_cleanup(monkeypatch): + from tools.nodes.common import depth_surface + + depth = np.broadcast_to(1 + 6 * (np.arange(128) % 16 < 8), (64, 128)) + with monkeypatch.context() as patch: + patch.setattr(depth_surface, "_remove_strip_faces", lambda group, geometry, **kw: geometry) + obj, set_value = panorama(depth) + set_value("Split Threshold", 1) + _, faces, _ = evaluated(obj) + edges = Counter(tuple(sorted((a, b))) for f in faces for a, b in zip(f, (*f[1:], f[0]))) + strips = 0 + for face in faces: + boundaries = [edges[tuple(sorted((a, b)))] == 1 for a, b in zip(face, (*face[1:], face[0]))] + strips += (boundaries[0] and boundaries[2]) or (boundaries[1] and boundaries[3]) + assert strips > 0 + obj, set_value = panorama(depth) + original_faces = evaluated(obj)[1] + set_value("Split Threshold", 1) + assert len(evaluated(obj)[1]) == len(faces) - strips + set_value("Split Threshold", 0) + assert evaluated(obj)[1] == original_faces diff --git a/tests/tools/nodes/test_inventory.py b/tests/tools/nodes/test_inventory.py index 72d4c17..33e40c9 100644 --- a/tests/tools/nodes/test_inventory.py +++ b/tests/tools/nodes/test_inventory.py @@ -18,6 +18,6 @@ def test_saved_asset_inventory(): if bpy.app.version >= (5, 0, 0): assert node_group.library is not None if name in {"O Image Depth Layer", "O Shadeless"}: - expected = ["Color", "Alpha", "Depth", "Bump Scale"] if name == "O Image Depth Layer" else ["Color"] + expected = ["Color", "Alpha", "Depth", "Bump Scale"] if name == "O Image Depth Layer" else ["Color", "Alpha"] assert [item.name for item in node_group.interface.items_tree if item.item_type == "SOCKET" and item.in_out == "INPUT"] == expected diff --git a/tests/tools/nodes/test_saved_layout.py b/tests/tools/nodes/test_saved_layout.py index c1063be..ab1bcc2 100644 --- a/tests/tools/nodes/test_saved_layout.py +++ b/tests/tools/nodes/test_saved_layout.py @@ -22,7 +22,7 @@ class NodeAssetsTest(unittest.TestCase): for group in target.node_groups if group.bl_idname == "GeometryNodeTree" ] - self.assertEqual(len(groups), 6) + self.assertEqual(len(groups), 7) for group in groups: validate_node_structure(group) positions = {node.name: tuple(node.location) for node in group.nodes diff --git a/tests/tools/nodes/test_shadeless.py b/tests/tools/nodes/test_shadeless.py new file mode 100644 index 0000000..26917e1 --- /dev/null +++ b/tests/tools/nodes/test_shadeless.py @@ -0,0 +1,52 @@ +"""Render the bundled Shadeless group's opacity and HDR color endpoints.""" + +import bpy +import numpy as np + +from anyimage.common.node import load_node_group + + +def test_shadeless_alpha_is_mixed_inside_the_group(tmp_path): + bpy.ops.wm.read_factory_settings(use_empty=True) + group = load_node_group("O Shadeless") + inputs = {s.name: s for s in group.interface.items_tree if s.item_type == "SOCKET" and s.in_out == "INPUT"} + assert list(inputs) == ["Color", "Alpha"] and inputs["Alpha"].default_value == 1 + scene = bpy.context.scene + scene.render.engine = "CYCLES" + scene.cycles.device = "CPU" + scene.cycles.samples = 8 + scene.cycles.use_denoising = False + scene.render.film_transparent = True + scene.render.resolution_x, scene.render.resolution_y = 96, 32 + scene.render.resolution_percentage = 100 + scene.render.image_settings.file_format = "OPEN_EXR" + scene.render.image_settings.color_mode = "RGBA" + scene.render.image_settings.color_depth = "32" + scene.render.filepath = str(tmp_path / "shadeless.exr") + bpy.ops.object.camera_add(location=(0, 0, 5)) + scene.camera = bpy.context.object + scene.camera.data.type = "ORTHO" + scene.camera.data.ortho_scale = 3 + for index, alpha in enumerate((0, 0.5, 1)): + bpy.ops.mesh.primitive_plane_add(size=1, location=(index - 1, 0, 0)) + material = bpy.data.materials.new(f"Alpha {alpha}") + material.use_nodes = True + material.node_tree.nodes.clear() + shader = material.node_tree.nodes.new("ShaderNodeGroup") + shader.node_tree = group + shader.inputs["Color"].default_value = (4, 2, 0.5, 1) + shader.inputs["Alpha"].default_value = alpha + output = material.node_tree.nodes.new("ShaderNodeOutputMaterial") + material.node_tree.links.new(shader.outputs[0], output.inputs["Surface"]) + bpy.context.object.data.materials.append(material) + bpy.ops.render.render(write_still=True) + image = bpy.data.images.load(scene.render.filepath, check_existing=False) + try: + pixels = np.array(image.pixels[:]).reshape(32, 96, 4) + for index, alpha in enumerate((0, 0.5, 1)): + patch = pixels[8:24, index * 32 + 8:index * 32 + 24] + np.testing.assert_allclose(patch[..., 3].mean(), alpha, atol=0.035) + np.testing.assert_allclose(pixels[16, 80, :3], (4, 2, 0.5), atol=1e-4) + finally: + bpy.data.images.remove(image) + bpy.ops.wm.read_factory_settings(use_empty=True) diff --git a/tests/tools/test_pack.py b/tests/tools/test_pack.py index af28f98..ee09293 100644 --- a/tests/tools/test_pack.py +++ b/tests/tools/test_pack.py @@ -101,6 +101,9 @@ class ExtensionPackagingTest(unittest.TestCase): "operators/convert_to_plane/object.py", "operators/convert_to_plane/operators.py", "operators/convert_to_plane/image_plane.py", + "operators/convert_to_panorama/__init__.py", + "operators/convert_to_panorama/operators.py", + "operators/convert_to_panorama/object.py", "operators/frame_tool.py", "operators/mask_tool.py", "operators/rectify_tool.py", @@ -124,6 +127,8 @@ class ExtensionPackagingTest(unittest.TestCase): self.assertIn("server/jobs/remove_background.py", names) self.assertIn("server/jobs/cutout.py", names) self.assertIn("server/jobs/depth_plane.py", names) + self.assertIn("server/jobs/panorama.py", names) + self.assertIn("server/geometry/panorama.py", names) self.assertIn("server/geometry/depth_texture.py", names) self.assertIn("server/geometry/prediction_artifacts.py", names) self.assertIn("server/geometry/edge_depth.py", names) @@ -138,7 +143,7 @@ class ExtensionPackagingTest(unittest.TestCase): self.assertNotIn("server/geometry/cutout.py", names) self.assertIn("server/models/ben2.py", names) self.assertNotIn("server/models/da3.py", names) - self.assertIn("server/models/moge2.py", names) + self.assertIn("server/models/moge.py", names) self.assertNotIn("server/jobs/debug_predictions.py", names) self.assertNotIn("operators/debug.py", names) self.assertFalse(any(name.startswith("server/outputs/") for name in names)) @@ -147,6 +152,7 @@ class ExtensionPackagingTest(unittest.TestCase): self.assertIn("server/models/onnx_ben2.py", names) self.assertNotIn("server/models/onnx_da3.py", names) self.assertIn("server/models/onnx_moge2.py", names) + self.assertIn("server/models/onnx_moge3.py", names) self.assertIn("server/models/onnx_upscale.py", names) self.assertIn("server/models/upscale.py", names) self.assertIn("server/__init__.py", names) diff --git a/tools/models/cli.py b/tools/models/cli.py index 147d28c..8ae06df 100644 --- a/tools/models/cli.py +++ b/tools/models/cli.py @@ -194,18 +194,19 @@ def prepare_model(model): if model.build: build_model(model) return - if model.key == "PISA_SR": + if model.key in {"PISA_SR", "MOGE3_VITL"}: model.destination.parent.mkdir(parents=True, exist_ok=True) - with TemporaryDirectory(prefix=".pisa-sr-", dir=MODELS_DIR) as directory: + exporter = "pisa_sr_export.py" if model.key == "PISA_SR" else "moge3_export.py" + with TemporaryDirectory(prefix=f".{model.folder}-", dir=MODELS_DIR) as directory: subprocess.run( - ["uv", "run", "--script", str(Path(__file__).with_name("pisa_sr_export.py")), + ["uv", "run", "--script", str(Path(__file__).with_name(exporter)), "--destination", directory], cwd=PROJECT_ROOT, check=True, ) models = [item for item in MODEL_FILES if item.key == model.key] for item in models: if not verify_file(Path(directory) / item.filename, item.size, item.sha256): - raise RuntimeError("PiSA-SR export differs from the published catalog") + raise RuntimeError(f"{model.key} export differs from the published catalog") for path in Path(directory).iterdir(): path.replace(model.destination.parent / path.name) return diff --git a/tools/models/moge3_export.py b/tools/models/moge3_export.py new file mode 100644 index 0000000..0285aec --- /dev/null +++ b/tools/models/moge3_export.py @@ -0,0 +1,167 @@ +# /// script +# requires-python = ">=3.11,<3.12" +# dependencies = [ +# "torch==2.13.0", "torchvision==0.28.0", "onnx==1.22.0", "numpy==2.4.6", +# "triton-windows==3.7.1.post27; sys_platform == 'win32'", +# "moge @ git+https://github.com/microsoft/MoGe.git@74fbce054ebed49800de42d0ad0e83495065719a", +# ] +# [tool.uv.sources] +# torch = { index = "pytorch" } +# torchvision = { index = "pytorch" } +# [[tool.uv.index]] +# name = "pytorch" +# url = "https://download.pytorch.org/whl/cu130" +# explicit = true +# /// +"""Export the dynamic MoGe-3 backbone and sparse refinement ONNX graphs.""" + +import argparse +import sys +from pathlib import Path + +import torch +import torch.nn.functional as F +from moge.model.v3 import MoGeModel +from moge.utils.geometry_torch import normalized_view_plane_uv + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src" / "anyimage")) +from server.models.onnx_moge3 import build_sparse_inputs + + +class Moge3Backbone(torch.nn.Module): + def __init__(self, model): + super().__init__() + self.model = model + + def forward(self, image, num_tokens): + m = self.model + aspect = image.shape[-1] / image.shape[-2] + bh = (num_tokens / aspect).sqrt().round().long() + bw = (num_tokens * aspect).sqrt().round().long() + feat, cls = m.encoder(image, bh, bw, return_class_token=True) + features = [] + for level in range(5): + uv = normalized_view_plane_uv( + width=bw * 2**level, + height=bh * 2**level, + aspect_ratio=aspect, + dtype=image.dtype, + device=image.device, + ).permute(2, 0, 1)[None] + features.append(torch.cat([feat, uv], 1) if level == 0 else uv) + neck = m.neck(features) + return ( + m.points_head(neck)[-1].permute(0, 2, 3, 1).float(), + features[0], + m.normal_head(neck)[-1], + m.mask_head(neck)[-1], + m.scale_head(cls), + ) + + +class Moge3Refiner(torch.nn.Module): + def __init__(self, r): + super().__init__() + self.r = r + self.n = len(r.down_stages) + + def conv(self, m, x, idx): + padded = torch.cat([x, torch.zeros_like(x[:1])], 0) + weights = m.weight.reshape(m.out_channels, 27, m.in_channels).permute(1, 2, 0) + result = padded[idx[:, 0]] @ weights[0] + for k in range(1, 27): + result = result + padded[idx[:, k]] @ weights[k] + return result + m.bias + + def stage(self, stage, x, idx): + for b in stage: + y = F.silu(b.norm1(x)) + y = F.silu(self.conv(b.conv1, y, idx)) + y = self.conv(b.conv2, y, idx) + x = y + b.skip_connection(x) + return x + + def forward(self, coord, encoder, *args): + n = self.n + ns = args[:n] + ps = args[n : 2 * n - 1] + us = args[2 * n - 1 : 3 * n - 2] + ei = args[-1] + r = self.r + x = r.input_proj(coord.reshape(-1, 3)) + skips = [] + for i, stage in enumerate(r.down_stages): + x = self.stage(stage, x, ns[i]) + if i < n - 1: + skips.append(x) + padded = torch.cat([x, torch.zeros_like(x[:1])], 0) + count = (ps[i] < x.shape[0]).sum(1, keepdim=True).to(x.dtype) + x = r.downsample_blocks[i].linear(padded[ps[i]].sum(1) / count) + ef = encoder.permute(0, 2, 3, 1).reshape(-1, encoder.shape[1])[ei] + x = r.fuse_proj(torch.cat([x, r.encoder_fuse(ef)], -1)) + x = self.stage(r.bottleneck_stage, x, ns[-1]) + for i, (up, stage) in enumerate(zip(r.upsample_blocks, r.up_stages)): + level = n - 2 - i + x = up.linear(x)[us[level]] + skips[level] + x = self.stage(stage, x, ns[level]) + return r.out_proj(x).reshape(coord.shape[0], coord.shape[1], coord.shape[2]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--checkpoint", default="Ruicheng/moge-3-vitl") + args = parser.parse_args() + args.destination.mkdir(parents=True, exist_ok=True) + checkpoint_options = ( + {"revision": "184008f877d7ad1ad4c2cd2182a9bd1f63d0e5be"} + if args.checkpoint == "Ruicheng/moge-3-vitl" + else {} + ) + model = ( + MoGeModel.from_pretrained(args.checkpoint, **checkpoint_options).eval().cuda() + ) + model.onnx_compatible_mode = True + backbone = Moge3Backbone(model).eval() + image = torch.zeros(1, 3, 192, 256, device="cuda") + with torch.no_grad(): + coord, encoder, *_ = backbone(image, torch.tensor(64, device="cuda")) + sparse = build_sparse_inputs(coord.cpu().numpy()) + indices = [torch.from_numpy(value).cuda() for value in sparse.values()] + names = ["coord", "encoder", *sparse] + axes = {name: {0: "count_" + name} for name in sparse} + axes.update( + coord={1: "height", 2: "width"}, + encoder={2: "token_height", 3: "token_width"}, + delta={1: "height", 2: "width"}, + ) + torch.onnx.export( + Moge3Refiner(model.refiner).eval(), + (coord, encoder, *indices), + str(args.destination / "refiner.onnx"), + input_names=names, + output_names=["delta"], + dynamic_axes=axes, + opset_version=18, + dynamo=False, + ) + torch.onnx.export( + backbone, + (image, torch.tensor(1200, device="cuda")), + str(args.destination / "backbone.onnx"), + input_names=["image", "num_tokens"], + output_names=["coord", "encoder", "normal", "mask", "scale"], + dynamic_axes={ + "image": {2: "height", 3: "width"}, + "coord": {1: "grid_height", 2: "grid_width"}, + "encoder": {2: "token_height", 3: "token_width"}, + "normal": {2: "grid_height", 3: "grid_width"}, + "mask": {2: "grid_height", 3: "grid_width"}, + }, + opset_version=18, + dynamo=False, + ) + + +if __name__ == "__main__": + main() diff --git a/tools/nodes/arrangement/dimensions.json b/tools/nodes/arrangement/dimensions.json index 0f84b54..cc712d0 100644 --- a/tools/nodes/arrangement/dimensions.json +++ b/tools/nodes/arrangement/dimensions.json @@ -30,6 +30,8 @@ "GeometryNodeFlipFaces": 0, "GeometryNodeGroup": 1, "GeometryNodeImageTexture": 2, + "GeometryNodeImageInfo": 0, + "GeometryNodeSetMaterial": 0, "GeometryNodeInputIndex": 0, "GeometryNodeInputMeshEdgeNeighbors": 0, "GeometryNodeInputMeshEdgeVertices": 0, diff --git a/tools/nodes/build.py b/tools/nodes/build.py index 13ffef6..6d85ec9 100644 --- a/tools/nodes/build.py +++ b/tools/nodes/build.py @@ -13,6 +13,7 @@ from .groups.image_depth_balloon import build_image_depth_balloon_group from .groups.image_layer import build_image_layer_group from .groups.image_depth_layer import build_image_depth_layer_group from .groups.shadeless import build_shadeless_group +from .groups.image_depth_panorama import build_image_depth_panorama_group OUTPUT_PATH = Path(__file__).resolve().parents[2] / "src/anyimage/assets/O_AnyImage.blend" @@ -22,6 +23,7 @@ def build_node_groups(): """Build each asset once in dependency order.""" image_plane = build_image_plane_group() return [ + build_image_depth_panorama_group(), image_plane, build_image_depth_plane_group(image_plane), build_image_relief_plane_group(image_plane), diff --git a/tools/nodes/check.py b/tools/nodes/check.py index c7f964d..158de06 100644 --- a/tools/nodes/check.py +++ b/tools/nodes/check.py @@ -7,6 +7,7 @@ import bpy ASSET_PATH = Path(__file__).resolve().parents[2] / "src/anyimage/assets/O_AnyImage.blend" GEOMETRY_NAMES = { + "O Image Depth Panorama", "O Image Plane", "O Image Depth Plane", "O Image Relief Plane", "O Image Cutout", "O Image Depth Cutout", "O Image Depth Balloon", } @@ -17,7 +18,7 @@ def check_asset_library(asset_path): """Check library completeness and finite, nonempty evaluated geometry.""" with bpy.data.libraries.load(str(asset_path), link=True) as (source, target): assert set(source.node_groups) == GEOMETRY_NAMES | SHADER_NAMES - target.node_groups = list(source.node_groups) + target.node_groups = sorted(GEOMETRY_NAMES | SHADER_NAMES) depth_image = bpy.data.images.new("Asset Check Depth", width=8, height=8, float_buffer=True) depth_image.colorspace_settings.name = "Non-Color" depth_image.pixels.foreach_set([ diff --git a/tools/nodes/common/depth_surface.py b/tools/nodes/common/depth_surface.py index 75ab342..94318cf 100644 --- a/tools/nodes/common/depth_surface.py +++ b/tools/nodes/common/depth_surface.py @@ -168,8 +168,8 @@ def _remove_strip_faces(group, geometry, *, triangles=False): return delete.outputs["Geometry"] -def split_depth_surface(group, geometry, image, strength, uniform, *, triangles=False): - """Keep each face's camera sample on its own corners before splitting.""" +def sample_face_camera(group, image): + """Sample camera XYZ at each planar face's UV center.""" nodes, links = group.nodes, group.links uv = nodes.new("GeometryNodeInputNamedAttribute") uv.data_type = "FLOAT_VECTOR" @@ -183,9 +183,17 @@ def split_depth_surface(group, geometry, image, strength, uniform, *, triangles= texture.extension = "EXTEND" links.new(image, texture.inputs["Image"]) links.new(face_uv.outputs["Value"], texture.inputs["Vector"]) - position = nodes.new("GeometryNodeInputPosition") - faces = _store(group, geometry, "face_center", position.outputs[0]) - faces = _store(group, faces, "face_camera", texture.outputs["Color"]) + return texture.outputs["Color"] + + +def split_depth_surface(group, geometry, face_camera, strength, uniform, *, + face_center=None, triangles=False): + """Split adjacent face samples using their Z depth and base center spacing.""" + nodes, links = group.nodes, group.links + if face_center is None: + face_center = nodes.new("GeometryNodeInputPosition").outputs[0] + faces = _store(group, geometry, "face_center", face_center) + faces = _store(group, faces, "face_camera", face_camera) axes = nodes.new("ShaderNodeSeparateXYZ") links.new(_read(group, "face_camera"), axes.inputs[0]) center_field = _read(group, "face_center") diff --git a/tools/nodes/common/material.py b/tools/nodes/common/material.py new file mode 100644 index 0000000..9bdc8ef --- /dev/null +++ b/tools/nodes/common/material.py @@ -0,0 +1,22 @@ +"""Carry the input mesh's material slots onto generated geometry.""" + + +def inherit_material_slots(group, geometry, source): + nodes, links = group.nodes, group.links + capture = nodes.new("GeometryNodeCaptureAttribute") + capture.domain = "POINT" + capture.capture_items.clear() + capture.capture_items.new("BOOLEAN", "Source") + capture.inputs["Source"].default_value = True + links.new(source, capture.inputs["Geometry"]) + join = nodes.new("GeometryNodeJoinGeometry") + links.new(geometry, join.inputs["Geometry"]) + links.new(capture.outputs["Geometry"], join.inputs["Geometry"]) + material = nodes.new("GeometryNodeSetMaterialIndex") + material.inputs["Material Index"].default_value = 0 + links.new(join.outputs["Geometry"], material.inputs["Geometry"]) + delete = nodes.new("GeometryNodeDeleteGeometry") + delete.domain = "POINT" + links.new(material.outputs["Geometry"], delete.inputs["Geometry"]) + links.new(capture.outputs["Source"], delete.inputs["Selection"]) + return delete.outputs["Geometry"] diff --git a/tools/nodes/groups/image_depth_cutout.py b/tools/nodes/groups/image_depth_cutout.py index 64e33bd..bf6ec64 100644 --- a/tools/nodes/groups/image_depth_cutout.py +++ b/tools/nodes/groups/image_depth_cutout.py @@ -5,7 +5,7 @@ from anyimage.operators.cutout_tool.shape import CUTOUT_Z_TO_X, NORMAL_SCALE_ATT from ..common.nodes import interface_socket, group_input, math_node, float_input, menu_switch, read_float_attribute, store_float_attribute, prepare_node_group from ..common.smoothing import expand_smooth from ..common.cutout import _island_cleanup, _position_field, _remove_zero_thickness, _shade_output -from ..common.depth_surface import project_depth_surface, store_surface_camera, remove_surface_attributes, split_depth_surface, thicken_depth_surface +from ..common.depth_surface import project_depth_surface, store_surface_camera, remove_surface_attributes, split_depth_surface, sample_face_camera, thicken_depth_surface from ..common.smoothing import edge_boundary_field @@ -436,7 +436,7 @@ def _build_depth_surface( nodes = group.nodes links = group.links geometry = split_depth_surface( - group, geometry, depth_image, split, uniform_scale, triangles=True + group, geometry, sample_face_camera(group, depth_image), split, uniform_scale, triangles=True ) original_camera = _position_field(group, depth_image) geometry, camera_position = store_surface_camera( diff --git a/tools/nodes/groups/image_depth_panorama.py b/tools/nodes/groups/image_depth_panorama.py new file mode 100644 index 0000000..fe7d827 --- /dev/null +++ b/tools/nodes/groups/image_depth_panorama.py @@ -0,0 +1,195 @@ +"""Build a quad panorama with periodic sampling and radial depth separation.""" + +import math + +import bpy + +from ..common.depth_surface import ( + _math, remove_surface_attributes, split_depth_surface, store_surface_camera, +) +from ..common.nodes import float_input, group_input, interface_socket, prepare_node_group +from ..common.material import inherit_material_slots +from ..common.smoothing import expand_smooth + + +def _quad_sphere(group, segments): + nodes, links = group.nodes, group.links + cube = nodes.new("GeometryNodeMeshCube") + cube.inputs["Size"].default_value = (2, 2, 2) + for axis in "XYZ": + links.new(segments, cube.inputs[f"Vertices {axis}"]) + xyz = nodes.new("ShaderNodeSeparateXYZ") + links.new(nodes.new("GeometryNodeInputPosition").outputs[0], xyz.inputs[0]) + squares = {axis: _math(group, "MULTIPLY", xyz.outputs[axis], xyz.outputs[axis]) for axis in "XYZ"} + combine = nodes.new("ShaderNodeCombineXYZ") + for axis, a, b in (("X", "Y", "Z"), ("Y", "Z", "X"), ("Z", "X", "Y")): + half = _math(group, "DIVIDE", _math(group, "ADD", squares[a], squares[b]), 2) + third = _math(group, "DIVIDE", _math(group, "MULTIPLY", squares[a], squares[b]), 3) + root = _math(group, "SQRT", _math(group, "ADD", _math(group, "SUBTRACT", 1, half), third)) + links.new(_math(group, "MULTIPLY", xyz.outputs[axis], root), combine.inputs[axis]) + position = nodes.new("GeometryNodeSetPosition") + links.new(cube.outputs["Mesh"], position.inputs["Geometry"]) + links.new(combine.outputs[0], position.inputs["Position"]) + return position.outputs["Geometry"] + + +def build_image_depth_panorama_group(): + group = bpy.data.node_groups.new("O Image Depth Panorama", "GeometryNodeTree") + group.is_modifier = group.use_fake_user = True + interface_socket(group, "Geometry", "INPUT", "NodeSocketGeometry") + subdivide = interface_socket(group, "Subdivide", "INPUT", "NodeSocketInt") + subdivide.default_value, subdivide.min_value, subdivide.max_value = 7, 0, 8 + float_input(group, "Depth Scale", 1, maximum=1, subtype="FACTOR") + float_input(group, "Split Threshold", 0, maximum=1, subtype="FACTOR") + smooth = interface_socket(group, "Smooth", "INPUT", "NodeSocketInt") + smooth.default_value, smooth.min_value, smooth.max_value = 0, 0, 20 + interface_socket(group, "Valid Only", "INPUT", "NodeSocketBool").default_value = True + options = group.interface.new_panel("Options", default_closed=True) + float_input(group, "Smooth Weight", 1, maximum=1, subtype="FACTOR", parent=options) + float_input(group, "Validity Threshold", 0.9, maximum=1, subtype="FACTOR", parent=options) + float_input(group, "Invalid Distance", 100, minimum=1e-5, subtype="DISTANCE", parent=options) + data = group.interface.new_panel("Data", default_closed=True) + interface_socket(group, "Depth Image", "INPUT", "NodeSocketImage", data).hide_in_modifier = True + float_input(group, "Reference Distance", 1, minimum=1e-6, subtype="DISTANCE", parent=data).hide_in_modifier = True + interface_socket(group, "Geometry", "OUTPUT", "NodeSocketGeometry") + nodes, links = group.nodes, group.links + + def read(name): + node = nodes.new("GeometryNodeInputNamedAttribute") + node.data_type = "FLOAT_VECTOR" + node.inputs["Name"].default_value = name + return node.outputs["Attribute"] + + def store(geometry, name, value, domain): + node = nodes.new("GeometryNodeStoreNamedAttribute") + node.data_type, node.domain = ("FLOAT2" if name == "UVMap" else "FLOAT_VECTOR"), domain + node.inputs["Name"].default_value = name + links.new(geometry, node.inputs["Geometry"]) + links.new(value, node.inputs["Value"]) + return node.outputs["Geometry"] + + def uv(direction): + normalize = nodes.new("ShaderNodeVectorMath") + normalize.operation = "NORMALIZE" + links.new(direction, normalize.inputs[0]) + xyz = nodes.new("ShaderNodeSeparateXYZ") + links.new(normalize.outputs[0], xyz.inputs[0]) + longitude = _math(group, "DIVIDE", _math(group, "ARCTAN2", xyz.outputs["Y"], xyz.outputs["X"]), 2 * math.pi) + z = _math(group, "MAXIMUM", -1, _math(group, "MINIMUM", 1, xyz.outputs["Z"])) + latitude = _math(group, "SUBTRACT", 1, _math(group, "DIVIDE", _math(group, "ARCCOSINE", z), math.pi)) + return longitude, latitude, xyz.outputs["Z"] + + def combine(u, v): + node = nodes.new("ShaderNodeCombineXYZ") + links.new(u, node.inputs["X"]) + links.new(v, node.inputs["Y"]) + return node.outputs[0] + + def texture(coordinates, interpolation): + inputs = group_input(nodes, {"Depth Image"}) + info = nodes.new("GeometryNodeImageInfo") + links.new(inputs.outputs["Depth Image"], info.inputs["Image"]) + axes = nodes.new("ShaderNodeSeparateXYZ") + links.new(coordinates, axes.inputs[0]) + half_pixel = _math(group, "DIVIDE", 0.5, info.outputs["Height"]) + v = _math(group, "MAXIMUM", half_pixel, + _math(group, "MINIMUM", axes.outputs["Y"], _math(group, "SUBTRACT", 1, half_pixel))) + node = nodes.new("GeometryNodeImageTexture") + node.interpolation, node.extension = interpolation, "REPEAT" + links.new(inputs.outputs["Depth Image"], node.inputs["Image"]) + links.new(combine(axes.outputs["X"], v), node.inputs["Vector"]) + return node.outputs["Color"] + + def depth(coordinates): + linear, nearest = texture(coordinates, "Linear"), texture(coordinates, "Closest") + alphas = [] + for color in (linear, nearest): + separate = nodes.new("FunctionNodeSeparateColor") + links.new(color, separate.inputs["Color"]) + alphas.append(separate.outputs["Alpha"]) + choose = nodes.new("GeometryNodeSwitch") + choose.input_type = "RGBA" + links.new(_math(group, "LESS_THAN", alphas[0], + _math(group, "SUBTRACT", alphas[1], 1e-6)), choose.inputs["Switch"]) + links.new(linear, choose.inputs["False"]) + links.new(nearest, choose.inputs["True"]) + return choose.outputs[0], alphas[1] + + inputs = group_input(nodes, {"Subdivide"}) + sphere = _quad_sphere(group, _math(group, "ADD", _math(group, "POWER", 2, inputs.outputs["Subdivide"]), 1)) + position = nodes.new("GeometryNodeInputPosition").outputs[0] + geometry = store(sphere, "o_panorama_direction", position, "POINT") + longitude, latitude, z = uv(read("o_panorama_direction")) + geometry = store(geometry, "o_panorama_uv", combine(longitude, latitude), "POINT") + face = nodes.new("GeometryNodeFieldOnDomain") + face.data_type, face.domain = "FLOAT_VECTOR", "FACE" + links.new(read("o_panorama_direction"), face.inputs["Value"]) + face_u, face_v, _ = uv(face.outputs["Value"]) + correction = _math(group, "ROUND", _math(group, "SUBTRACT", face_u, longitude)) + choose = nodes.new("GeometryNodeSwitch") + choose.input_type = "FLOAT" + links.new(_math(group, "GREATER_THAN", _math(group, "ABSOLUTE", z), 0.999999), choose.inputs["Switch"]) + links.new(_math(group, "ADD", longitude, correction), choose.inputs["False"]) + links.new(face_u, choose.inputs["True"]) + geometry = store(geometry, "UVMap", combine(choose.outputs[0], latitude), "CORNER") + pixel, _ = depth(read("o_panorama_uv")) + geometry = store(geometry, "o_panorama_distance", pixel, "POINT") + nearest = nodes.new("FunctionNodeSeparateColor") + links.new(texture(read("o_panorama_uv"), "Closest"), nearest.inputs["Color"]) + controls = group_input(nodes, {"Validity Threshold"}) + invalid = _math(group, "MAXIMUM", + _math(group, "LESS_THAN", nearest.outputs["Alpha"], controls.outputs["Validity Threshold"]), + _math(group, "LESS_THAN", nearest.outputs["Red"], 1e-5)) + geometry = store(geometry, "o_panorama_invalid", invalid, "POINT") + face_invalid = read("o_panorama_invalid") + any_invalid = _math(group, "GREATER_THAN", face_invalid, 0) + mixed = _math(group, "MULTIPLY", any_invalid, + _math(group, "LESS_THAN", face_invalid, 1)) + selection = nodes.new("GeometryNodeSwitch") + selection.input_type = "BOOLEAN" + links.new(group_input(nodes, {"Valid Only"}).outputs["Valid Only"], selection.inputs["Switch"]) + links.new(mixed, selection.inputs["False"]) + links.new(any_invalid, selection.inputs["True"]) + delete = nodes.new("GeometryNodeDeleteGeometry") + delete.domain, delete.mode = "FACE", "ALL" + links.new(geometry, delete.inputs["Geometry"]) + links.new(selection.outputs[0], delete.inputs["Selection"]) + controls = group_input(nodes, {"Split Threshold", "Reference Distance"}) + strength = controls.outputs["Split Threshold"] + geometry = split_depth_surface(group, delete.outputs[0], depth(combine(face_u, face_v))[0], strength, + _math(group, "DIVIDE", 1, controls.outputs["Reference Distance"]), + face_center=face.outputs["Value"]) + geometry, distance = store_surface_camera(group, geometry, read("o_panorama_distance"), strength) + axes = nodes.new("ShaderNodeSeparateXYZ") + links.new(distance, axes.inputs[0]) + distance = nodes.new("GeometryNodeSwitch") + distance.input_type = "FLOAT" + links.new(read("o_panorama_invalid"), distance.inputs["Switch"]) + links.new(axes.outputs["X"], distance.inputs["False"]) + links.new(group_input(nodes, {"Invalid Distance"}).outputs["Invalid Distance"], distance.inputs["True"]) + controls = group_input(nodes, {"Depth Scale"}) + radius = _math(group, "ADD", 1, _math(group, "MULTIPLY", controls.outputs["Depth Scale"], _math(group, "SUBTRACT", distance.outputs[0], 1))) + radial = nodes.new("ShaderNodeVectorMath") + radial.operation = "SCALE" + links.new(read("o_panorama_direction"), radial.inputs[0]) + links.new(radius, radial.inputs[3]) + deform = nodes.new("GeometryNodeSetPosition") + links.new(geometry, deform.inputs["Geometry"]) + links.new(radial.outputs[0], deform.inputs["Position"]) + geometry = remove_surface_attributes(group, deform.outputs[0]) + remove = nodes.new("GeometryNodeRemoveAttribute") + remove.pattern_mode = "WILDCARD" + remove.inputs["Name"].default_value = "o_panorama_*" + links.new(geometry, remove.inputs["Geometry"]) + geometry = expand_smooth(group, remove.outputs[0], iterations_name="Smooth", weight_name="Smooth Weight") + flip = nodes.new("GeometryNodeFlipFaces") + links.new(geometry, flip.inputs["Mesh"]) + smooth = nodes.new("GeometryNodeSetShadeSmooth") + smooth.domain = "FACE" + smooth.inputs["Shade Smooth"].default_value = True + links.new(flip.outputs["Mesh"], smooth.inputs["Geometry"]) + geometry = inherit_material_slots(group, smooth.outputs[0], group_input(nodes, {"Geometry"}).outputs["Geometry"]) + output = nodes.new("NodeGroupOutput") + links.new(geometry, output.inputs["Geometry"]) + prepare_node_group(group) + return group diff --git a/tools/nodes/groups/image_depth_plane.py b/tools/nodes/groups/image_depth_plane.py index 1d34364..887ce3d 100644 --- a/tools/nodes/groups/image_depth_plane.py +++ b/tools/nodes/groups/image_depth_plane.py @@ -7,7 +7,7 @@ from ..common.nodes import prepare_node_group, float_input from ..common.smoothing import expand_smooth from ..common.depth_surface import ( project_depth_surface, thicken_depth_surface, remove_surface_attributes, - split_depth_surface, store_surface_camera, + split_depth_surface, sample_face_camera, store_surface_camera, ) DEPTH_PLANE_NODE_GROUP_NAME = "O Image Depth Plane" @@ -63,7 +63,7 @@ def build_image_depth_plane_group(image_plane_group): smooth_weight.min_value, smooth_weight.max_value = 0.0, 1.0 smooth_weight.subtype = "FACTOR" validity_threshold = _interface_socket(group, "Validity Threshold", "INPUT", "NodeSocketFloat", parent=options) - validity_threshold.default_value = 0.9 + validity_threshold.default_value = 0.1 validity_threshold.min_value, validity_threshold.max_value = 0.0, 1.0 validity_threshold.subtype = "FACTOR" data = group.interface.new_panel(name="Data", default_closed=True) @@ -111,13 +111,10 @@ def build_image_depth_plane_group(image_plane_group): invalid.data_type, invalid.operation = "FLOAT", "LESS_THAN" links.new(threshold_input.outputs["Validity Threshold"], invalid.inputs["B"]) links.new(channels.outputs["Alpha"], invalid.inputs["A"]) - points = nodes.new("GeometryNodeFieldOnDomain") - points.data_type, points.domain = "BOOLEAN", "POINT" - links.new(invalid.outputs["Result"], points.inputs["Value"]) delete = nodes.new("GeometryNodeDeleteGeometry") delete.domain, delete.mode = "FACE", "ALL" links.new(plane.outputs["Geometry"], delete.inputs["Geometry"]) - links.new(points.outputs["Value"], delete.inputs["Selection"]) + links.new(invalid.outputs["Result"], delete.inputs["Selection"]) cull_controls = _group_input(nodes, {"Valid Only"}) cull = nodes.new("GeometryNodeSwitch") cull.input_type = "GEOMETRY" @@ -126,7 +123,7 @@ def build_image_depth_plane_group(image_plane_group): links.new(plane.outputs["Geometry"], cull.inputs["False"]) split_input = _group_input(nodes, {"Split Threshold"}) geometry = split_depth_surface( - group, cull.outputs["Output"], texture_input.outputs["Depth Image"], + group, cull.outputs["Output"], sample_face_camera(group, texture_input.outputs["Depth Image"]), split_input.outputs["Split Threshold"], scale_input.outputs["Uniform Scale"], ) geometry, camera = store_surface_camera( diff --git a/tools/nodes/groups/image_plane.py b/tools/nodes/groups/image_plane.py index 12152e2..3b6d964 100644 --- a/tools/nodes/groups/image_plane.py +++ b/tools/nodes/groups/image_plane.py @@ -3,6 +3,7 @@ import bpy from ..common.nodes import prepare_node_group +from ..common.material import inherit_material_slots IMAGE_PLANE_GROUP_NAME = "O Image Plane" @@ -218,23 +219,8 @@ def build_image_plane_group(): links.new(place_grid.outputs["Geometry"], geometry_switch.inputs["False"]) links.new(place_cube.outputs["Geometry"], geometry_switch.inputs["True"]) - capture_source = nodes.new("GeometryNodeCaptureAttribute") - capture_source.domain = "POINT" - capture_source.capture_items.clear() - capture_source.capture_items.new("BOOLEAN", "Source") - capture_source.inputs["Source"].default_value = True - links.new(geometry_input.outputs["Geometry"], capture_source.inputs["Geometry"]) - join_material = nodes.new("GeometryNodeJoinGeometry") - links.new(geometry_switch.outputs["Output"], join_material.inputs["Geometry"]) - links.new(capture_source.outputs["Geometry"], join_material.inputs["Geometry"]) - set_material_index = nodes.new("GeometryNodeSetMaterialIndex") - set_material_index.inputs["Material Index"].default_value = 0 - links.new(join_material.outputs["Geometry"], set_material_index.inputs["Geometry"]) - delete_source = nodes.new("GeometryNodeDeleteGeometry") - delete_source.domain = "POINT" - links.new(set_material_index.outputs["Geometry"], delete_source.inputs["Geometry"]) - links.new(capture_source.outputs["Source"], delete_source.inputs["Selection"]) - links.new(delete_source.outputs["Geometry"], output.inputs["Geometry"]) + geometry = inherit_material_slots(group, geometry_switch.outputs["Output"], geometry_input.outputs["Geometry"]) + links.new(geometry, output.inputs["Geometry"]) group.color_tag = "GEOMETRY" group.description = ( diff --git a/tools/nodes/groups/shadeless.py b/tools/nodes/groups/shadeless.py index d68db3c..3f4b84a 100644 --- a/tools/nodes/groups/shadeless.py +++ b/tools/nodes/groups/shadeless.py @@ -14,6 +14,9 @@ def build_shadeless_group(): group.interface.new_socket( name="Color", in_out="INPUT", socket_type="NodeSocketColor" ) + alpha = group.interface.new_socket(name="Alpha", in_out="INPUT", socket_type="NodeSocketFloat") + alpha.default_value, alpha.min_value, alpha.max_value = 1.0, 0.0, 1.0 + alpha.subtype = "FACTOR" nodes = group.nodes links = group.links group_input = nodes.new("NodeGroupInput") @@ -67,7 +70,15 @@ def build_shadeless_group(): links.new(shader_factor.outputs[0], mix.inputs["Fac"]) links.new(diffuse.outputs["BSDF"], mix.inputs[1]) links.new(emission.outputs["Emission"], mix.inputs[2]) - links.new(mix.outputs["Shader"], group_output.inputs["Shader"]) + transparent = nodes.new("ShaderNodeBsdfTransparent") + transparent.location = (550.0, -200.0) + alpha_mix = nodes.new("ShaderNodeMixShader") + alpha_mix.location = (850.0, 100.0) + group_output.location = (1100.0, 100.0) + links.new(group_input.outputs["Alpha"], alpha_mix.inputs["Fac"]) + links.new(transparent.outputs["BSDF"], alpha_mix.inputs[1]) + links.new(mix.outputs["Shader"], alpha_mix.inputs[2]) + links.new(alpha_mix.outputs["Shader"], group_output.inputs["Shader"]) group.color_tag = "SHADER" group.description = ( "Display color without scene lighting while preserving shadows "