icrdr/AnyImage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add comprehensive documentation for SnapForm extension

Browse Source 
- Created development.md for development setup and licensing information.
- Added features.md detailing functionalities and usage instructions for developers and users.
- Introduced index.md to serve as a navigation hub for the documentation.
- Developed outputs.md to explain output formats and their specifications.
- Established service.md to describe the AI Service architecture and environment setup.
- Created surface.md to outline the process of converting Image Empty to perspective meshes.
This commit is contained in:
马南
2026-08-13 23:57:09 +08:00
parent caeb6d706c
commit 13fab9ce88
28 changed files with 998 additions and 830 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
src/snapform/image_mesh.py
View File

@@ -7,6 +7,7 @@ from .image_depth import (
create_material,
draw_process_resolution,
is_image_empty,
normal_data_name,
plane_bounds,
prepare_image_input,
process_resolution_scale_property,
@@ -18,22 +19,16 @@ from .image_depth import (
POINT_ATTRIBUTES = {
"o_depth": ("FLOAT", "point_depth"),
"o_confidence": ("FLOAT", "point_confidence"),
"o_confidence_percentile": (
"FLOAT",
"point_confidence_percentile",
),
"o_background": ("BOOLEAN", "point_background"),
}
FACE_ATTRIBUTES = {
"o_face_confidence": (
"FLOAT",
"face_confidence_percentile",
),
"o_background_face": ("BOOLEAN", "face_background"),
}
OPTIONAL_FACE_ATTRIBUTES = {
"o_mask": ("BOOLEAN", "face_mask"),
"o_depth_discontinuity": ("FLOAT", "face_depth_discontinuity"),
}
NODE_GROUP_NAME = "O DA3 Surface"
MOGE_NODE_GROUP_NAME = "O MoGe2 Surface"
@@ -61,10 +56,10 @@ def load_image_mesh_data(npz_path, moge2=False):
"point_depth",
]
if moge2:
required.append("face_mask")
required.extend(("face_mask", "face_depth_discontinuity", "normal_map"))
else:
required.extend(
("point_confidence", "point_confidence_percentile", "point_background", "face_confidence_percentile", "face_background")
("face_confidence_percentile", "face_depth_discontinuity")
)
missing = [key for key in required if key not in archive.files]
if missing:
@@ -93,9 +88,9 @@ def load_image_mesh_data(npz_path, moge2=False):
for _name, (_data_type, key) in OPTIONAL_FACE_ATTRIBUTES.items():
if key in archive.files:
attributes[key] = np.asarray(archive[key])
if "vertex_normal" in archive.files:
attributes["vertex_normal"] = np.asarray(
archive["vertex_normal"],
if "normal_map" in archive.files:
attributes["normal_map"] = np.asarray(
archive["normal_map"],
dtype=np.float32,
)
except OSError as error:
@@ -112,6 +107,14 @@ def load_image_mesh_data(npz_path, moge2=False):
raise ValueError(f"Unsupported image size: {image_size}")
if not reference_depth > 0.0:
raise ValueError("Surface reference depth must be positive")
if moge2 and attributes["normal_map"].shape != (
image_size[1],
image_size[0],
3,
):
raise ValueError("Surface normal map and image sizes do not match")
if moge2 and not np.isfinite(attributes["normal_map"]).all():
raise ValueError("Surface normal map contains invalid values")
for _name, (_data_type, key) in POINT_ATTRIBUTES.items():
if key in attributes and len(attributes[key]) != len(vertices):
raise ValueError(f"Point attribute has the wrong length: {key}")
@@ -208,6 +211,13 @@ def assign_snapform_attributes(mesh, attributes, moge2=False):
if moge2:
assign_mesh_attribute(mesh, "o_depth", "FLOAT", "POINT", attributes["point_depth"])
assign_mesh_attribute(mesh, "o_mask", "BOOLEAN", "FACE", attributes["face_mask"])
assign_mesh_attribute(
mesh,
"o_depth_discontinuity",
"FLOAT",
"FACE",
attributes["face_depth_discontinuity"],
)
return
for name, (data_type, key) in POINT_ATTRIBUTES.items():
assign_mesh_attribute(
@@ -234,50 +244,12 @@ def assign_snapform_attributes(mesh, attributes, moge2=False):
"FACE",
attributes[key],
)
assign_edge_depth_difference(mesh, attributes["point_depth"])
def assign_edge_depth_difference(mesh, point_depth):
import numpy as np
edge_vertices = np.empty(len(mesh.edges) * 2, dtype=np.int32)
mesh.edges.foreach_get("vertices", edge_vertices)
edge_vertices = edge_vertices.reshape(-1, 2)
depth = np.asarray(point_depth, dtype=np.float32)
first = depth[edge_vertices[:, 0]]
second = depth[edge_vertices[:, 1]]
denominator = np.maximum(np.minimum(np.abs(first), np.abs(second)), 1e-6)
difference = np.abs(first - second) / denominator
assign_mesh_attribute(
mesh,
"o_edge_stretch",
"o_depth_discontinuity",
"FLOAT",
"EDGE",
difference,
)
def remove_surface_background(vertices, faces, uv, attributes):
"""Remove background faces and compact matching point/face data."""
import numpy as np
background_faces = np.asarray(attributes["face_background"], dtype=bool)
kept_faces = faces[~background_faces]
if not len(kept_faces):
raise ValueError("Background removal deleted every surface face")
used_vertices = np.unique(kept_faces)
index_map = np.full(len(vertices), -1, dtype=np.int32)
index_map[used_vertices] = np.arange(len(used_vertices), dtype=np.int32)
filtered_attributes = {}
for _name, (_data_type, key) in POINT_ATTRIBUTES.items():
filtered_attributes[key] = attributes[key][used_vertices]
for _name, (_data_type, key) in FACE_ATTRIBUTES.items():
filtered_attributes[key] = attributes[key][~background_faces]
return (
vertices[used_vertices],
index_map[kept_faces],
uv[used_vertices],
filtered_attributes,
"FACE",
attributes["face_depth_discontinuity"],
)
@@ -285,7 +257,7 @@ def prepare_moge_surface_archive(results_path, output_path):
import numpy as np
with np.load(results_path, allow_pickle=False) as archive:
required = {"depth", "mask", "points", "intrinsics"}
required = {"depth", "mask", "normal", "points", "intrinsics"}
missing = required.difference(archive.files)
if missing:
raise ValueError(f"MoGe-2 surface data is missing: {', '.join(sorted(missing))}")
@@ -293,11 +265,7 @@ def prepare_moge_surface_archive(results_path, output_path):
mask = np.asarray(archive["mask"], dtype=bool)
points = np.asarray(archive["points"], dtype=np.float32)
intrinsics = np.asarray(archive["intrinsics"], dtype=np.float32)
normal = (
np.asarray(archive["normal"], dtype=np.float32)
if "normal" in archive.files
else None
)
normal = np.asarray(archive["normal"], dtype=np.float32)
if depth.shape[0] != 1 or points.shape[:3] != (*depth.shape,):
raise ValueError("Convert to Surface requires one matching MoGe-2 point grid")
raw_depth = depth[0]
@@ -314,8 +282,9 @@ def prepare_moge_surface_archive(results_path, output_path):
if not len(faces):
raise ValueError("MoGe-2 output does not contain a valid surface")
flat_points = points[0].reshape(-1, 3)
clean_mask = mask[0] & ~depth_discontinuity_mask(raw_depth, 0.04)
face_mask = clean_mask.reshape(-1)[faces].all(axis=1)
face_mask = mask[0].reshape(-1)[faces].all(axis=1)
depth_edge_ratio = depth_discontinuity_ratio(raw_depth)
face_depth_discontinuity = depth_edge_ratio.reshape(-1)[faces].max(axis=1)
used = np.unique(faces)
index_map = np.full(height * width, -1, dtype=np.int32)
index_map[used] = np.arange(len(used), dtype=np.int32)
@@ -341,32 +310,21 @@ def prepare_moge_surface_archive(results_path, output_path):
image_size=np.array((width, height), dtype=np.int32),
reference_depth=np.float32(reference_depth),
point_depth=point_depth,
vertex_normal=(
normal[0].reshape(-1, 3)[used]
if normal is not None
else np.empty((0, 3), dtype=np.float32)
),
normal_map=normal[0],
face_mask=face_mask,
face_depth_discontinuity=face_depth_discontinuity,
)
return output_path
def depth_discontinuity_mask(depth, threshold):
def depth_discontinuity_ratio(depth):
import numpy as np
edges = np.zeros(depth.shape, dtype=bool)
for axis in (0, 1):
first = np.take(depth, range(depth.shape[axis] - 1), axis=axis)
second = np.take(depth, range(1, depth.shape[axis]), axis=axis)
scale = np.maximum(np.minimum(np.abs(first), np.abs(second)), 1e-6)
discontinuity = np.abs(first - second) / scale > threshold
lower = [slice(None), slice(None)]
upper = [slice(None), slice(None)]
lower[axis] = slice(0, -1)
upper[axis] = slice(1, None)
edges[tuple(lower)] |= discontinuity
edges[tuple(upper)] |= discontinuity
return edges
padded = np.pad(depth, 1, mode="edge")
windows = np.lib.stride_tricks.sliding_window_view(padded, (3, 3))
difference = np.nanmax(windows, axis=(-2, -1))
difference -= np.nanmin(windows, axis=(-2, -1))
return difference / np.maximum(depth, 1e-6)
def node_asset_path():
@@ -505,7 +463,6 @@ def build_image_mesh(
delete_source,
create_camera,
unshaded,
remove_background=False,
moge2=False,
):
import numpy as np
@@ -519,13 +476,6 @@ def build_image_mesh(
reference_depth,
attributes,
) = load_image_mesh_data(npz_path, moge2=moge2)
if remove_background:
camera_vertices, faces, uv, attributes = remove_surface_background(
camera_vertices,
faces,
uv,
attributes,
)
vertices, camera_parameters = empty_anchored_vertices(
source_object,
camera_vertices,
@@ -541,24 +491,25 @@ def build_image_mesh(
mesh.update()
for polygon in mesh.polygons:
polygon.use_smooth = True
if moge2 and len(attributes.get("vertex_normal", ())) == len(vertices):
normals = attributes["vertex_normal"].copy()
normals[:, 0] /= camera_parameters["scale_x"]
normals[:, 1] /= -camera_parameters["scale_y"]
normals[:, 2] /= -camera_parameters["scale_z"]
lengths = np.linalg.norm(normals, axis=1, keepdims=True)
normals /= np.maximum(lengths, 1e-6)
set_custom_normals = getattr(
mesh,
"normals_split_custom_set_from_vertices",
None,
)
if callable(set_custom_normals):
set_custom_normals(normals.tolist())
mesh_object = bpy.data.objects.new(source_name, mesh)
context.collection.objects.link(mesh_object)
mesh_object.matrix_world = source_object.matrix_world.copy()
depth_scale = material_depth_scale(mesh_object, camera_parameters)
normal_image = None
if moge2 and attributes.get("normal_map", np.empty(0)).size:
from .image_depth import create_moge_normal_image
normal_map = attributes["normal_map"].copy()
normal_map[..., 0] /= camera_parameters["scale_x"]
normal_map[..., 1] /= -camera_parameters["scale_y"]
normal_map[..., 2] /= -camera_parameters["scale_z"]
lengths = np.linalg.norm(normal_map, axis=-1, keepdims=True)
normal_map /= np.maximum(lengths, 1e-6)
normal_image = create_moge_normal_image(
normal_data_name(source_object),
normal_map,
convert_coordinates=False,
)
material = create_material(
source_name,
source_object.data,
@@ -567,6 +518,7 @@ def build_image_mesh(
surface_group_name=(
MOGE_SURFACE_MATERIAL_NODE_GROUP_NAME if moge2 else None
),
normal_image=normal_image,
)
mesh.materials.append(material)
mesh_object["snapform_reference_depth"] = reference_depth
@@ -620,10 +572,6 @@ class BuildImageMesh(bpy.types.Operator):
default=False,
options={"HIDDEN", "SKIP_SAVE"},
)
snapform_remove_background: bpy.props.BoolProperty(
default=False,
options={"HIDDEN", "SKIP_SAVE"},
)
snapform_moge2: bpy.props.BoolProperty(
default=False,
options={"HIDDEN", "SKIP_SAVE"},
@@ -642,7 +590,6 @@ class BuildImageMesh(bpy.types.Operator):
self.snapform_delete_source,
self.snapform_create_camera,
self.snapform_unshaded,
self.snapform_remove_background,
self.snapform_moge2,
)
except (OSError, RuntimeError, ValueError) as error:
@@ -658,7 +605,7 @@ class ConvertImageToMesh(bpy.types.Operator):
bl_options = {"REGISTER"}
snapform_model: bpy.props.EnumProperty(
name="Model",
name="Depth From",
description="Model used to estimate image depth",
items=image_surface_model_items,
default=1,