From 102e1bfa9b25ecb250aa9b27d293015d2670286b Mon Sep 17 00:00:00 2001 From: nan Date: Fri, 14 Aug 2026 21:57:08 +0800 Subject: [PATCH] feat: Update documentation and improve depth processing logic; bump version to 0.11.1 --- AGENTS.md | 6 +++ README.md | 4 +- docs/depth-plane.md | 2 +- docs/features.md | 2 +- docs/service.md | 11 ++++ docs/surface.md | 2 +- pyproject.toml | 2 +- src/snapform/blender_manifest.toml | 2 +- src/snapform/da3_onnx.py | 21 ++++++++ src/snapform/image_depth.py | 72 ++++++++++++++++++++++---- src/snapform/image_mesh.py | 7 ++- src/snapform/worker.py | 27 +++++++--- tools/validate_depth_plane_asset.py | 17 ++++-- tools/validate_geometry_nodes_asset.py | 8 ++- 14 files changed, 149 insertions(+), 34 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 80c2ef1..31e5c2a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,12 @@ - 说明性内容（例如提交说明、文档、注释、mermaid），以中文为主，专有名词、约定俗称可用英文 +## 项目文档 + +- 动手前先读 `docs/index.md`，它是全部文档的入口，按主题链接并帮助理解项目结构与代码逻辑 +- 主题映射：功能与使用 `docs/features.md`、Depth Plane `docs/depth-plane.md`、Surface `docs/surface.md`、输出格式 `docs/outputs.md`、AI Service `docs/service.md`、开发与许可证 `docs/development.md` +- 修改代码或执行方式后，同步更新对应主题文档与 `README.md`，保持文档与实现一致 + ## Markdown 规范 - 不使用`---`分割器 diff --git a/README.md b/README.md index 01377f4..6e36d3c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ SnapForm 是一个 Blender 扩展，可以把一张照片、一段视频或一 最快的 DirectML/CoreML provider，否则 CPU。下载超时会自动重试国内镜像，无需手动配置 5. 打开 3D 视图右侧的 `SnapForm` 面板，点击 `Install Environment` -扩展只内置对应平台的 uv。首次使用会自动下载轻量的 ONNX Runtime 环境，随后自动下载 `MoGe-2 ViT-B Normal` 和 +扩展只内置对应平台的 uv。首次使用会自动下载轻量的 ONNX Runtime 环境，随后自动下载 `MoGe-2 ViT-S Normal` 和 `BEN2 Base`；不安装 PyTorch，也不下载各模型的 Python 源码包。其他 AI 模型可在 `Models` panel 中按需下载，请保持网络畅通，首次安装需要一些时间。 @@ -45,7 +45,7 @@ SnapForm 是一个 Blender 扩展，可以把一张照片、一段视频或一 ### 转换为透视网格 选中单张图片的 Image Empty，右键打开 `SnapForm` 子菜单并选择 `Convert to Surface`。可选择 MoGe-2 -ViT-S、ViT-B、ViT-L 或 DA3 Small、Base，其中 MoGe-2 ViT-B 为默认模型。SnapForm 会按模型输出的 +ViT-S、ViT-B、ViT-L 或 DA3 Small、Base，其中 MoGe-2 ViT-S 为默认模型。SnapForm 会按模型输出的 相机内参与三维点建立透视网格，并匹配原 Empty 的可见尺寸、Offset 与 Transform。MoGe-2 使用内置 NumPy 网格构建与深度断层 mask 生成干净拓扑；DA3 保留 confidence 与边拉伸剔除。两者再由独立的 Geometry Nodes 修改器完成剔面、快速重拓扑、Relax 与 UV 重投。开启 Remove Background 时，背景三角面会在转换期间 diff --git a/docs/depth-plane.md b/docs/depth-plane.md index 640882f..4170cd7 100644 --- a/docs/depth-plane.md +++ b/docs/depth-plane.md @@ -14,7 +14,7 @@ - `MoGe-2`：使用所选模型输出 metric depth、normal、validity mask 与 normalized intrinsics。Depth Plane 不保存体积较大的 points；Surface 会在 AI 进程内用内置 NumPy 网格构建生成 `mesh.npz`。Debug 始终保留原始 `results.npz`，并按 Output Format 单独生成 Maps、纹理 GLB 或彩色 PLY。 - `Original`：直接把原图作为深度贴图，不显示估算参数。仅在开启 `Remove Background` 时运行 BEN2 取得前景 Alpha；视频或序列帧按需同时抽取原始帧与前景序列，把原始帧作为 `.depth` 序列。 - `Original` 是默认深度来源。选择 `DA3` 时，其 Model 与 Resolution Scale + `MoGe-2 ViT-S` 是默认深度来源。选择 `DA3` 时，其 Model 与 Resolution Scale 参数会集中显示在独立边框中。`Remove Background` 与 `Unshaded` 直接显示，不再使用 Options 折叠； `Keep Original` 默认关闭，并在分割线下方保持为最后一项。仅当开启 `Remove Background` 时才需要 `BEN2 Base`，并把 `.foreground` 的 Alpha 作为独立 diff --git a/docs/features.md b/docs/features.md index 6818cc9..8823bed 100644 --- a/docs/features.md +++ b/docs/features.md @@ -51,7 +51,7 @@ 1. 先运行 `uv run python prepare_vendor.py`，按 `vendor.lock.toml` 下载并校验固定版本的 第三方构建输入；再运行 `uv run python pack.py`，分别构建 `windows-x64`、`macos-arm64`、 - `linux-x64` 三个平台的 Extension ZIP（如 `SnapForm.v0.11.0.macos-arm64.zip`）。 + `linux-x64` 三个平台的 Extension ZIP（如 `SnapForm.v0.11.1.macos-arm64.zip`）。 每个包只含对应平台的 uv 二进制，不再携带 AI 项目的源码包。 2. 在 Blender 的 `Preferences > Extensions` 中选择 `Install from Disk`。 3. 在 Extension Preferences 中确认 Storage Root 与 Device。Device 是正式功能 diff --git a/docs/service.md b/docs/service.md index d2332c2..0ff70b7 100644 --- a/docs/service.md +++ b/docs/service.md @@ -59,3 +59,14 @@ GitHub 不参与用户安装流程。`prepare_vendor.py` 只为扩展打包准 Service 全局同时执行一个 AI 任务。BEN2 独立缓存，深度模型只保留最近使用的一个；切换模型时释放旧会话。取消任务不会重启 Service，ONNX Runtime 当前调用返回后在最近检查点结束。 启动与崩溃信息写入 `~/.snapform/service.log`。 + +## 已知限制 + +DA3 ONNX 的注意力是朴素实现（`MatMul → Softmax → MatMul`），峰值显存与 `(帧数 × 每帧 patch 数)²` 成正比。迁移到 ONNX 前的 PyTorch 版使用 `scaled_dot_product_attention`（`XFORMERS_DISABLED=1`），按块计算注意力分数、不落地完整注意力矩阵，显存近似线性，因此视频能处理几十上百帧；ONNX 导出把 SDPA 回退成朴素注意力，多帧在中高分辨率下会显存爆炸（约 12 帧 @ 1008 即需 ~90 GB）。 + +作为兜底，`da3_onnx.infer` 在推理前按 `views × patches` 预估注意力张量，超过 4 GiB 直接报错并提示减少帧数或 Resolution Scale，避免 onnxruntime 报出难以定位的分配失败或 `UnicodeDecodeError`（DirectML 在中文 Windows 下报错信息为 GBK 编码，被 UTF-8 解码掩盖）。 + +待解决方向： + +- 重新导出 DA3 ONNX，使用 ORT 的 `Attention` / `MultiHeadAttention` 贡献算子，或注入内存高效的注意力算子； +- 或把视频按小窗口分批推理，再按 `ref_view_strategy` 对齐相机位姿拼接。 diff --git a/docs/surface.md b/docs/surface.md index b6fc842..1be19e0 100644 --- a/docs/surface.md +++ b/docs/surface.md @@ -5,7 +5,7 @@ 1. 在 3D View 选中仅包含一帧的 Image Empty，右键打开 `SnapForm` 子菜单并选择 `Convert to Surface`。该入口独立于 `Convert to Depth Plane`，当前不处理视频或序列帧。 - 模型列表包含三个 MoGe-2 Normal 模型和两个 DA3 模型，默认使用 MoGe-2 ViT-B Normal。DA3 的 + 模型列表包含三个 MoGe-2 Normal 模型和两个 DA3 模型，默认使用 MoGe-2 ViT-S Normal。DA3 的 Resolution Scale 与 Depth Plane 使用相同设计；MoGe-2 改为显示共享的 Resolution Level。 2. Service 请求 DA3 的 `depth`、`conf` 与 `intrinsics`。仅在开启 `Remove Background` 时运行 BEN2 取得 前景 Alpha，并在输出前删除包含背景顶点的三角面；关闭时不加载 BEN2。Depth 不做 smoothing，Service diff --git a/pyproject.toml b/pyproject.toml index f8e2df3..19a00e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "snapform" -version = "0.11.0" +version = "0.11.1" description = "Fast image-guided structure building tools for Blender" readme = "README.md" requires-python = ">=3.11" diff --git a/src/snapform/blender_manifest.toml b/src/snapform/blender_manifest.toml index 4f80b2b..c2ed409 100644 --- a/src/snapform/blender_manifest.toml +++ b/src/snapform/blender_manifest.toml @@ -1,7 +1,7 @@ schema_version = "1.0.0" id = "snapform" -version = "0.11.0" +version = "0.11.1" name = "SnapForm" tagline = "Build spatial structures from images, fast" maintainer = "OmooLab" diff --git a/src/snapform/da3_onnx.py b/src/snapform/da3_onnx.py index 20c211d..cc89a61 100644 --- a/src/snapform/da3_onnx.py +++ b/src/snapform/da3_onnx.py @@ -15,6 +15,10 @@ except ImportError: ONNX_PATH = "model.onnx" PATCH_SIZE = 14 +# DA3 cross-view attention is quadratic in (views x patches); the peak +# attention tensor alone can exhaust memory long before ONNX Runtime reports +# a generic allocation failure. +ATTENTION_BUDGET_BYTES = 4 * 1024 * 1024 * 1024 STRATEGY_CODES = { "first": 0, "middle": 1, @@ -177,6 +181,14 @@ def camera_from_pose(pose_encoding, height, width): return extrinsics, intrinsics +def estimate_attention_bytes(views, height, width): + """Estimate the peak cross-view attention tensor size in bytes.""" + patches = (height // PATCH_SIZE) * (width // PATCH_SIZE) + tokens = views * patches + # 12 attention heads in DA3 Base; float32 activations. + return tokens * tokens * 12 * 4 + + def infer( session, image_paths, @@ -192,6 +204,15 @@ def infer( process_res, process_res_method, ) + _, views, _, height, width = model_input.shape + attention_bytes = estimate_attention_bytes(views, height, width) + if attention_bytes > ATTENTION_BUDGET_BYTES: + raise RuntimeError( + f"DA3 cannot process {views} frame(s) at {width}x{height}: " + "the cross-view attention would need about " + f"{attention_bytes // 2**30} GB. Reduce the frame count or " + "the Resolution Scale." + ) strategy_code = np.asarray( [STRATEGY_CODES.get(ref_view_strategy, 2)], dtype=np.int64, diff --git a/src/snapform/image_depth.py b/src/snapform/image_depth.py index d72401d..2ddb810 100644 --- a/src/snapform/image_depth.py +++ b/src/snapform/image_depth.py @@ -1538,6 +1538,53 @@ def modifier_input_identifier(node_group, name): raise RuntimeError(f"Geometry Nodes input is missing: {name}") +def modifier_input_slots(modifier): + """Return the runtime input collection, or None for the legacy API.""" + properties = getattr(modifier, "properties", None) + if properties is None: + return None + return getattr(properties, "inputs", None) + + +def set_modifier_input(modifier, identifier, value): + """Assign a value to a Geometry Nodes modifier input socket.""" + inputs = modifier_input_slots(modifier) + if inputs is not None: + # Blender 5.2+ exposes inputs as runtime RNA keyed by socket identifier. + getattr(inputs, identifier).value = value + else: + # Blender 4.x exposes inputs as modifier ID properties. + modifier[identifier] = value + + +def get_modifier_input(modifier, identifier): + """Read a Geometry Nodes modifier input socket value.""" + inputs = modifier_input_slots(modifier) + if inputs is not None: + return getattr(inputs, identifier).value + return modifier[identifier] + + +def modifier_input_driver_path(modifier, identifier): + """Return the driver data path for a modifier input socket.""" + if modifier_input_slots(modifier) is not None: + return ( + f'modifiers["{modifier.name}"]' + f".properties.inputs.{identifier}.value" + ) + return f'modifiers["{modifier.name}"]["{identifier}"]' + + +def drive_modifier_input(owner_object, modifier, identifier, expression="frame"): + """Add a scripted driver to a modifier input so it follows the scene frame.""" + driver = owner_object.driver_add( + modifier_input_driver_path(modifier, identifier) + ) + driver.driver.type = "SCRIPTED" + driver.driver.expression = expression + return driver + + def add_depth_plane_processing( plane_object, depth_image, @@ -1565,12 +1612,10 @@ def add_depth_plane_processing( ("Cyclic", use_cyclic), ): identifier = modifier_input_identifier(node_group, name) - modifier[identifier] = value + set_modifier_input(modifier, identifier, value) if animate_frame: frame_identifier = modifier_input_identifier(node_group, "Frame") - driver = modifier.driver_add(f'["{frame_identifier}"]') - driver.driver.type = "SCRIPTED" - driver.driver.expression = "frame" + drive_modifier_input(plane_object, modifier, frame_identifier) plane_object.update_tag(refresh={"DATA"}) return modifier @@ -1601,12 +1646,11 @@ def add_moge_depth_plane_processing( ("Frames", max(int(frame_duration or 1), 1)), ("Cyclic", use_cyclic), ): - modifier[modifier_input_identifier(node_group, name)] = value + identifier = modifier_input_identifier(node_group, name) + set_modifier_input(modifier, identifier, value) if animate_frame: frame_identifier = modifier_input_identifier(node_group, "Frame") - driver = modifier.driver_add(f'["{frame_identifier}"]') - driver.driver.type = "SCRIPTED" - driver.driver.expression = "frame" + drive_modifier_input(plane_object, modifier, frame_identifier) plane_object.update_tag(refresh={"DATA"}) return modifier @@ -1882,7 +1926,9 @@ class BuildDepthPlane(bpy.types.Operator): assign_to_source=False, ) color_image = None - if self.snapform_color_frames_dir: + # The foreground result already carries the color, so the original + # frame sequence is unused when background removal is available. + if self.snapform_color_frames_dir and foreground_image is None: color_image = create_sequence_image( f"{image_base_name(source_object.data)}.frame", self.snapform_color_frames_dir, @@ -2029,7 +2075,9 @@ class BuildMoge2DepthPlane(bpy.types.Operator): assign_to_source=False, ) color_image = None - if self.snapform_color_frames_dir: + # The foreground result already carries the color, so the original + # frame sequence is unused when background removal is available. + if self.snapform_color_frames_dir and foreground_image is None: color_image = create_sequence_image( f"{image_base_name(source_object.data)}.frame", self.snapform_color_frames_dir, @@ -2098,7 +2146,9 @@ class BuildOriginalDepthPlane(bpy.types.Operator): assign_to_source=False, ) color_image = None - if self.snapform_color_frames_dir: + # The foreground result already carries the color, so the original + # frame sequence is unused when background removal is available. + if self.snapform_color_frames_dir and foreground_image is None: color_image = create_sequence_image( f"{image_base_name(source_object.data)}.frame", self.snapform_color_frames_dir, diff --git a/src/snapform/image_mesh.py b/src/snapform/image_mesh.py index 3c4e3b3..cc3fff9 100644 --- a/src/snapform/image_mesh.py +++ b/src/snapform/image_mesh.py @@ -13,6 +13,7 @@ from .image_depth import ( process_resolution_scale_property, remove_temporary_input, scaled_process_resolution, + set_modifier_input, video_total_frames, ) @@ -417,7 +418,11 @@ def add_geometry_nodes_processing( modifier.node_group = node_group modifier.show_group_selector = False identifier = modifier_input_identifier(node_group, "Resolution") - modifier[identifier] = default_remesh_resolution(vertices, faces) + set_modifier_input( + modifier, + identifier, + default_remesh_resolution(vertices, faces), + ) def create_matching_camera( diff --git a/src/snapform/worker.py b/src/snapform/worker.py index 70e3131..d04e6a3 100644 --- a/src/snapform/worker.py +++ b/src/snapform/worker.py @@ -2016,14 +2016,25 @@ def reconstruct(args, model_cache=None, cancel_check=None): 0.25, f"Estimating depth for {len(images)} frame(s)", ) - prediction = infer( - model, - [str(path) for path in images], - process_res=args.process_res, - process_res_method=args.process_res_method, - use_ray_pose=bool(args.use_ray_pose), - ref_view_strategy=args.ref_view_strategy, - ) + try: + prediction = infer( + model, + [str(path) for path in images], + process_res=args.process_res, + process_res_method=args.process_res_method, + use_ray_pose=bool(args.use_ray_pose), + ref_view_strategy=args.ref_view_strategy, + ) + except UnicodeDecodeError as error: + # DirectML reports failures with a locale-specific message that + # ONNX Runtime cannot always decode, masking the real error. + if provider_label(model) == "DirectML": + raise RuntimeError( + "DA3 inference failed on DirectML. Switch the device to " + "CPU in SnapForm Preferences and retry. " + f"Original error: {error}" + ) from error + raise check_cancelled(cancel_check) write_status(args.status, 0.88, f"Exporting {args.export_format}") if args.export_format == "mesh": diff --git a/tools/validate_depth_plane_asset.py b/tools/validate_depth_plane_asset.py index 8ac4544..967fb0e 100644 --- a/tools/validate_depth_plane_asset.py +++ b/tools/validate_depth_plane_asset.py @@ -11,8 +11,11 @@ from snapform.image_depth import ( SURFACE_MATERIAL_NODE_GROUP_NAME, add_depth_plane_processing, create_material, + get_modifier_input, load_depth_plane_node_group, + modifier_input_driver_path, modifier_input_identifier, + set_modifier_input, ) from snapform.image_mesh import load_geometry_node_group @@ -275,24 +278,28 @@ def main(): material, animate_frame=True, ) - modifier[modifier_input_identifier(group, "Depth Scale")] = -2.0 + set_modifier_input( + modifier, + modifier_input_identifier(group, "Depth Scale"), + -2.0, + ) for name, expected in ( ("Frame Start", 1), ("Frames", 1), ("Cyclic", False), ): - assert modifier[modifier_input_identifier(group, name)] == expected + identifier = modifier_input_identifier(group, name) + assert get_modifier_input(modifier, identifier) == expected assert not any( node.bl_idname == "GeometryNodeInputSceneTime" for node in group.nodes ) frame_identifier = modifier_input_identifier(group, "Frame") + frame_path = modifier_input_driver_path(modifier, frame_identifier) frame_driver = next( fcurve for fcurve in obj.animation_data.drivers - if fcurve.data_path == ( - f'modifiers["{modifier.name}"]["{frame_identifier}"]' - ) + if fcurve.data_path == frame_path ) assert frame_driver.driver.type == "SCRIPTED" assert frame_driver.driver.expression == "frame" diff --git a/tools/validate_geometry_nodes_asset.py b/tools/validate_geometry_nodes_asset.py index 4439ba8..179457a 100644 --- a/tools/validate_geometry_nodes_asset.py +++ b/tools/validate_geometry_nodes_asset.py @@ -7,6 +7,10 @@ import numpy as np sys.path.insert(0, str(Path(__file__).parents[1] / "src")) +from snapform.image_depth import ( + get_modifier_input, + set_modifier_input, +) from snapform.image_mesh import create_matching_camera, load_geometry_node_group @@ -18,7 +22,7 @@ def set_input(modifier, group, name, value): and item.in_out == "INPUT" and item.name == name ) - modifier[item.identifier] = value + set_modifier_input(modifier, item.identifier, value) def add_face_attributes(mesh, count): @@ -151,7 +155,7 @@ def main(): for item in group.interface.items_tree if item.item_type == "SOCKET" and item.name == "Depth Continuity" ) - assert modifier[discontinuity_item.identifier] == 100.0 + assert get_modifier_input(modifier, discontinuity_item.identifier) == 100.0 obj.update_tag(refresh={"DATA"}) bpy.context.view_layer.update() culled = obj.evaluated_get(bpy.context.evaluated_depsgraph_get())