Add SnapForm panels, preferences, properties, and runtime management

- Implemented RuntimeModelsPanel and DebugPanel for UI integration in Blender.
- Created SnapFormPreferences to manage user settings for storage and runtime configuration.
- Defined SnapFormSettings to hold various properties related to input, model selection, and export options.
- Developed runtime management functions for installation, model downloading, and reconstruction commands.
- Added status bar functionality to display job progress and status in the Blender UI.
- Implemented worker script for handling model downloads and depth reconstruction tasks.
- Introduced unit tests for download progress reporting and image depth extraction functionalities.
This commit is contained in:
2026-08-03 10:13:33 +08:00
parent 9586b35440
commit b02372c457
26 changed files with 1849 additions and 181 deletions

640
src/snapform/image_depth.py Normal file
View File

@@ -0,0 +1,640 @@
from pathlib import Path
import bpy
DEPTH_KEY_CANDIDATES = ("depth", "depths")
INTRINSICS_KEY_CANDIDATES = ("intrinsics", "ixts")
IMAGE_DEPTH_MODEL_ITEMS = (
(
"SMALL",
"DA3 Small",
"Fastest model with Apache 2.0 weights",
),
(
"BASE",
"DA3 Base",
"Balanced model with Apache 2.0 weights",
),
(
"LARGE_1_1",
"DA3 Large 1.1",
"Higher quality model with CC BY-NC 4.0 weights",
),
(
"GIANT_1_1",
"DA3 Giant 1.1",
"Giant model with Gaussian head and CC BY-NC 4.0 weights",
),
(
"NESTED_GIANT_LARGE_1_1",
"DA3 Nested Giant Large 1.1",
"Nested Giant model with metric depth and CC BY-NC 4.0 weights",
),
)
DISPLACEMENT_STRENGTH = -1.0
DISPLACEMENT_MIDLEVEL = 1.0
BUMP_BASE_DISTANCE = 0.001
def is_image_empty(candidate):
return bool(
candidate
and getattr(candidate, "type", None) == "EMPTY"
and getattr(candidate, "empty_display_type", None) == "IMAGE"
and getattr(candidate, "data", None) is not None
)
def image_path(image):
filepath_from_user = getattr(image, "filepath_from_user", None)
filepath = (
filepath_from_user()
if callable(filepath_from_user)
else getattr(image, "filepath", "")
)
if not filepath:
return Path()
return Path(bpy.path.abspath(filepath)).expanduser()
def plane_bounds(source_object):
image_width, image_height = source_object.data.size
if image_width <= 0 or image_height <= 0:
raise ValueError("The source image has no readable dimensions")
aspect = image_width / image_height
display_size = float(source_object.empty_display_size)
if aspect >= 1.0:
width = display_size
height = display_size / aspect
else:
width = display_size * aspect
height = display_size
offset = source_object.empty_image_offset
x_min = float(offset[0]) * width
y_min = float(offset[1]) * height
return (
x_min,
x_min + width,
y_min,
y_min + height,
)
def plane_fit_transform(source_object, camera_bounds):
target_bounds = plane_bounds(source_object)
camera_width = camera_bounds[1] - camera_bounds[0]
camera_height = camera_bounds[3] - camera_bounds[2]
if camera_width <= 0.0 or camera_height <= 0.0:
raise ValueError("The camera plane has invalid dimensions")
target_width = target_bounds[1] - target_bounds[0]
target_height = target_bounds[3] - target_bounds[2]
scale_x = target_width / camera_width
scale_y = target_height / camera_height
scale_z = (scale_x * scale_y) ** 0.5
camera_center_x = (camera_bounds[0] + camera_bounds[1]) * 0.5
camera_center_y = (camera_bounds[2] + camera_bounds[3]) * 0.5
target_center_x = (target_bounds[0] + target_bounds[1]) * 0.5
target_center_y = (target_bounds[2] + target_bounds[3]) * 0.5
return (
scale_x,
scale_y,
scale_z,
target_center_x - camera_center_x * scale_x,
target_center_y - camera_center_y * scale_y,
)
def apply_plane_transform(plane_object, source_object, camera_bounds):
from mathutils import Matrix
scale_x, scale_y, scale_z, offset_x, offset_y = plane_fit_transform(
source_object,
camera_bounds,
)
fit_matrix = Matrix(
(
(scale_x, 0.0, 0.0, offset_x),
(0.0, scale_y, 0.0, offset_y),
(0.0, 0.0, scale_z, 0.0),
(0.0, 0.0, 0.0, 1.0),
)
)
plane_object.matrix_world = source_object.matrix_world @ fit_matrix
return abs(float(plane_object.scale.z))
def extract_depth_array(npz_path):
try:
import numpy as np
except ImportError as error:
raise RuntimeError("Blender Python does not provide NumPy") from error
try:
with np.load(npz_path, allow_pickle=False) as archive:
depth_key = next(
(
key
for key in DEPTH_KEY_CANDIDATES
if key in archive.files
),
None,
)
if depth_key is None:
keys = ", ".join(archive.files) or "none"
raise ValueError(
f"Mini NPZ has no depth array; available keys: {keys}"
)
depth = np.asarray(archive[depth_key], dtype=np.float32)
except OSError as error:
raise ValueError(f"Unable to read Mini NPZ: {npz_path}") from error
if depth.ndim >= 3 and depth.shape[-1] == 1:
depth = depth[..., 0]
if depth.ndim == 3:
depth = depth[0]
if depth.ndim != 2:
raise ValueError(
f"Unsupported depth array shape: {tuple(depth.shape)}"
)
return depth
def extract_intrinsics_array(npz_path):
try:
import numpy as np
except ImportError as error:
raise RuntimeError("Blender Python does not provide NumPy") from error
try:
with np.load(npz_path, allow_pickle=False) as archive:
intrinsics_key = next(
(
key
for key in INTRINSICS_KEY_CANDIDATES
if key in archive.files
),
None,
)
if intrinsics_key is None:
keys = ", ".join(archive.files) or "none"
raise ValueError(
"Mini NPZ has no camera intrinsics; "
f"available keys: {keys}"
)
intrinsics = np.asarray(
archive[intrinsics_key],
dtype=np.float32,
)
except OSError as error:
raise ValueError(f"Unable to read Mini NPZ: {npz_path}") from error
if intrinsics.ndim == 3:
intrinsics = intrinsics[0]
if intrinsics.shape != (3, 3):
raise ValueError(
f"Unsupported intrinsics shape: {tuple(intrinsics.shape)}"
)
return intrinsics
def prepare_depth_values(depth):
import numpy as np
valid = np.isfinite(depth) & (depth > 0.0)
if not valid.any():
raise ValueError("Depth array contains no positive finite values")
values = depth.astype(np.float32, copy=True)
values[~valid] = 0.0
return values
def camera_plane_bounds(
source_object,
depth,
intrinsics,
displacement_strength,
):
import numpy as np
scale = abs(float(displacement_strength))
if scale <= 1e-8:
raise ValueError("Displacement Strength must not be zero")
valid = np.isfinite(depth) & (depth > 0.0)
if not valid.any():
raise ValueError("Depth array contains no positive finite values")
reference_depth = float(np.median(depth[valid]))
reference_distance = reference_depth * scale
height, width = depth.shape
fx = float(intrinsics[0, 0])
fy = float(intrinsics[1, 1])
cx = float(intrinsics[0, 2])
cy = float(intrinsics[1, 2])
if fx <= 0.0 or fy <= 0.0:
raise ValueError("Camera intrinsics contain an invalid focal length")
source_width, source_height = source_object.data.size
if source_width <= 0 or source_height <= 0:
raise ValueError("The source image has no readable dimensions")
source_aspect = source_width / source_height
# Predicted fx/fy may differ even for square pixels. Preserve their projected
# area while enforcing the source image aspect ratio to avoid image stretch.
raw_width = width * reference_distance / fx
raw_height = height * reference_distance / fy
plane_area = raw_width * raw_height
plane_width = float(np.sqrt(plane_area * source_aspect))
plane_height = float(np.sqrt(plane_area / source_aspect))
x_min = -(cx / width) * plane_width
x_max = (1.0 - cx / width) * plane_width
y_min = -(1.0 - cy / height) * plane_height
y_max = (cy / height) * plane_height
# Preserve an intentional Image Empty origin offset around the camera frame.
offset = source_object.empty_image_offset
x_offset = (float(offset[0]) + 0.5) * plane_width
y_offset = (float(offset[1]) + 0.5) * plane_height
return (
(
x_min + x_offset,
x_max + x_offset,
y_min + y_offset,
y_max + y_offset,
),
reference_depth,
)
def create_depth_image(name, depth_values):
import numpy as np
height, width = depth_values.shape
image = bpy.data.images.new(
name,
width=width,
height=height,
alpha=False,
float_buffer=True,
)
image.colorspace_settings.name = "Non-Color"
image.file_format = "OPEN_EXR"
# NPZ arrays use a top-left origin, while Blender image pixels start below.
depth_pixels = np.flipud(depth_values)
rgba = np.empty((height, width, 4), dtype=np.float32)
rgba[..., 0] = depth_pixels
rgba[..., 1] = depth_pixels
rgba[..., 2] = depth_pixels
rgba[..., 3] = 1.0
image.pixels.foreach_set(rgba.ravel())
# Pack as EXR so values outside 0..1 survive saving and reopening the blend.
image.pack()
return image
def grid_segments(bounds, subdivision_level):
if subdivision_level < 0:
raise ValueError("Subdivision must not be negative")
width = bounds[1] - bounds[0]
height = bounds[3] - bounds[2]
if width <= 0.0 or height <= 0.0:
raise ValueError("The plane has invalid dimensions")
short_segments = 2**subdivision_level
if width >= height:
return (
max(short_segments, int(short_segments * width / height + 0.5)),
short_segments,
)
return (
short_segments,
max(short_segments, int(short_segments * height / width + 0.5)),
)
def create_plane_mesh(source_object, bounds=None, subdivision_level=5):
x_min, x_max, y_min, y_max = (
bounds if bounds is not None else plane_bounds(source_object)
)
x_segments, y_segments = grid_segments(
(x_min, x_max, y_min, y_max),
subdivision_level,
)
vertices = []
uv_coordinates = []
for y_index in range(y_segments + 1):
v = y_index / y_segments
y = y_min + (y_max - y_min) * v
for x_index in range(x_segments + 1):
u = x_index / x_segments
x = x_min + (x_max - x_min) * u
vertices.append((x, y, 0.0))
uv_coordinates.append((u, v))
faces = []
row_width = x_segments + 1
for y_index in range(y_segments):
for x_index in range(x_segments):
bottom_left = y_index * row_width + x_index
faces.append(
(
bottom_left,
bottom_left + 1,
bottom_left + row_width + 1,
bottom_left + row_width,
)
)
mesh = bpy.data.meshes.new(f"{source_object.name}.DA3Depth")
mesh.from_pydata(vertices, (), faces)
mesh.update()
for polygon in mesh.polygons:
polygon.use_smooth = True
uv_layer = mesh.uv_layers.new(name="UVMap")
for loop in mesh.loops:
uv_layer.data[loop.index].uv = uv_coordinates[
loop.vertex_index
]
return mesh
def create_material(
name,
color_image,
depth_scale,
):
material = bpy.data.materials.new(name)
material.use_nodes = True
if hasattr(material, "surface_render_method"):
material.surface_render_method = "DITHERED"
nodes = material.node_tree.nodes
nodes.clear()
output = nodes.new("ShaderNodeOutputMaterial")
shader = nodes.new("ShaderNodeBsdfPrincipled")
color_texture = nodes.new("ShaderNodeTexImage")
bump = nodes.new("ShaderNodeBump")
roughness_curve = nodes.new("ShaderNodeFloatCurve")
color_texture.image = color_image
color_texture.interpolation = "Linear"
color_texture.extension = "EXTEND"
bump.inputs["Distance"].default_value = BUMP_BASE_DISTANCE * depth_scale
shader.inputs["IOR"].default_value = 1.2
roughness_curve.mapping.initialize()
curve_points = roughness_curve.mapping.curves[0].points
curve_points[0].location = (0.0, 0.5)
curve_points[-1].location = (1.0, 0.1)
roughness_curve.mapping.update()
color_texture.location = (-640.0, 180.0)
roughness_curve.location = (-300.0, 320.0)
bump.location = (-300.0, -80.0)
shader.location = (80.0, 180.0)
output.location = (460.0, 180.0)
links = material.node_tree.links
links.new(color_texture.outputs["Color"], shader.inputs["Base Color"])
links.new(color_texture.outputs["Alpha"], shader.inputs["Alpha"])
links.new(color_texture.outputs["Color"], bump.inputs["Height"])
links.new(bump.outputs["Normal"], shader.inputs["Normal"])
links.new(
color_texture.outputs["Color"],
roughness_curve.inputs["Value"],
)
links.new(
roughness_curve.outputs["Value"],
shader.inputs["Roughness"],
)
links.new(shader.outputs["BSDF"], output.inputs["Surface"])
return material
def build_depth_plane(
context,
source_object,
npz_path,
subdivision_level,
delete_source,
):
source_name = source_object.name
depth = extract_depth_array(npz_path)
intrinsics = extract_intrinsics_array(npz_path)
depth_values = prepare_depth_values(depth)
bounds, reference_depth = camera_plane_bounds(
source_object,
depth_values,
intrinsics,
DISPLACEMENT_STRENGTH,
)
depth_name = f"{source_name}.depth"
depth_image = create_depth_image(
depth_name,
depth_values,
)
mesh = create_plane_mesh(source_object, bounds, subdivision_level)
plane_object = bpy.data.objects.new(
source_name,
mesh,
)
context.collection.objects.link(plane_object)
depth_scale = apply_plane_transform(
plane_object,
source_object,
bounds,
)
subdivision = plane_object.modifiers.new("Subdivision", "SUBSURF")
subdivision.subdivision_type = "SIMPLE"
subdivision.levels = 1
subdivision.render_levels = 2
texture = bpy.data.textures.new(
depth_name,
type="IMAGE",
)
texture.image = depth_image
texture.extension = "EXTEND"
displace = plane_object.modifiers.new("Displace", "DISPLACE")
displace.texture = texture
displace.texture_coords = "UV"
displace.direction = "NORMAL"
displace.strength = DISPLACEMENT_STRENGTH
displace.mid_level = DISPLACEMENT_MIDLEVEL
material = create_material(
f"{source_name}.DA3DepthMaterial",
source_object.data,
depth_scale,
)
mesh.materials.append(material)
plane_object["snapform_reference_depth"] = reference_depth
plane_object["snapform_camera_plane_width"] = bounds[1] - bounds[0]
plane_object["snapform_camera_plane_height"] = bounds[3] - bounds[2]
plane_object["snapform_depth_scale"] = depth_scale
for selected in context.selected_objects:
selected.select_set(False)
plane_object.select_set(True)
context.view_layer.objects.active = plane_object
if delete_source:
bpy.data.objects.remove(source_object, do_unlink=True)
plane_object.name = source_name
return plane_object
class BuildDepthPlane(bpy.types.Operator):
bl_idname = "snapform.build_depth_plane"
bl_label = "Build Depth Plane"
bl_options = {"INTERNAL"}
snapform_source_object_name: bpy.props.StringProperty(
options={"HIDDEN", "SKIP_SAVE"},
)
snapform_npz_path: bpy.props.StringProperty(
options={"HIDDEN", "SKIP_SAVE"},
)
snapform_subdivision_level: bpy.props.IntProperty(
default=5,
min=0,
max=10,
options={"HIDDEN", "SKIP_SAVE"},
)
snapform_delete_source: bpy.props.BoolProperty(
default=True,
options={"HIDDEN", "SKIP_SAVE"},
)
def execute(self, context):
source_object = bpy.data.objects.get(self.snapform_source_object_name)
if not is_image_empty(source_object):
self.report(
{"ERROR"},
"The source image empty is no longer available",
)
return {"CANCELLED"}
npz_path = Path(self.snapform_npz_path)
if not npz_path.is_file():
self.report(
{"ERROR"},
f"DA3 Mini NPZ does not exist: {npz_path}",
)
return {"CANCELLED"}
try:
build_depth_plane(
context,
source_object,
npz_path,
self.snapform_subdivision_level,
self.snapform_delete_source,
)
except (OSError, RuntimeError, ValueError) as error:
self.report({"ERROR"}, str(error))
return {"CANCELLED"}
return {"FINISHED"}
class ConvertImageToDepthPlane(bpy.types.Operator):
bl_idname = "snapform.convert_image_to_depth_plane"
bl_label = "Convert to Depth Plane"
bl_description = "Run DA3 and convert the image to a displaced depth plane"
bl_options = {"REGISTER"}
snapform_model: bpy.props.EnumProperty(
name="Model",
description="DA3 model used to estimate image depth",
items=IMAGE_DEPTH_MODEL_ITEMS,
default="BASE",
)
snapform_use_ray_pose: bpy.props.BoolProperty(
name="Use Ray Pose",
description="Use ray-based camera pose estimation",
default=True,
)
snapform_process_res: bpy.props.IntProperty(
name="Process Resolution",
description="DA3 processing resolution",
default=1024,
min=14,
max=2016,
step=14,
)
snapform_delete_original: bpy.props.BoolProperty(
name="Delete Original",
description="Delete the source Image Empty after conversion",
default=True,
)
snapform_subdivision_level: bpy.props.IntProperty(
name="Subdivision",
description="Power of two for the short-edge segment count; the long edge follows the image aspect ratio",
default=5,
min=0,
max=10,
)
@classmethod
def poll(cls, context):
from .operators import active_job
return is_image_empty(context.object) and active_job() is None
def invoke(self, context, _event):
return context.window_manager.invoke_props_dialog(self, width=340)
def draw(self, _context):
layout = self.layout
layout.prop(self, "snapform_model")
layout.prop(self, "snapform_use_ray_pose")
layout.prop(self, "snapform_process_res")
layout.prop(self, "snapform_subdivision_level")
layout.prop(self, "snapform_delete_original")
def execute(self, context):
from .operators import IMAGE_DEPTH_JOB
source_object = context.object
source_path = image_path(source_object.data)
if not source_path.is_file():
self.report(
{"ERROR"},
"The image empty must reference an existing image file",
)
return {"CANCELLED"}
result = bpy.ops.snapform.run_job(
job_type=IMAGE_DEPTH_JOB,
snapform_input_path=str(source_path),
snapform_source_object_name=source_object.name,
snapform_model_key=self.snapform_model,
snapform_use_ray_pose=self.snapform_use_ray_pose,
snapform_process_res=self.snapform_process_res,
snapform_subdivision_level=self.snapform_subdivision_level,
snapform_delete_source=self.snapform_delete_original,
)
if "CANCELLED" in result:
return {"CANCELLED"}
return {"FINISHED"}
def draw_image_context_menu(self, context):
active_object = context.object
if not is_image_empty(active_object):
return
self.layout.separator()
row = self.layout.row()
row.operator_context = "INVOKE_DEFAULT"
row.operator(
"snapform.convert_image_to_depth_plane",
text="Convert to Depth Plane",
icon="MOD_DISPLACE",
)