Add new functionality for image rectification and depth metadata handling

- Implemented a preview tool for rectifying images with interactive aspect controls in `preview.py`.
- Added tests for depth metadata loading to ensure consistent schema usage in `test_depth_metadata.py`.
- Developed comprehensive tests for cutout calibration processes in `test_cutout_calibration.py`, including stability checks and depth calibration.
- Created tests for frame compositing operations in `test_frame_compositing.py`, ensuring correct handling of depth and transparency.
- Introduced material slot inheritance functionality in `material_slots.py` to manage geometry material assignments.
- Built shared shader nodes for material processing in `shader.py`, including roughness and bump mapping setups.
This commit is contained in:
2026-09-12 19:20:06 +08:00
parent e8268504c2
commit 938d95db0f
175 changed files with 3812 additions and 6864 deletions

View File

@@ -5,6 +5,7 @@ from dataclasses import dataclass
from ...preferences import addon_preferences
from ...common.image import image_empty_bounds
from . import balloon
@dataclass(frozen=True)
@@ -661,568 +662,6 @@ def triangulate_selection_contours(
return (*result, components) if return_components else result
DEFAULT_INFLATION = 0.45
def distances_to_contour(points, contour):
import numpy as np
points = np.asarray(points, dtype=np.float64)
contour = np.asarray(contour, dtype=np.float64)
squared = np.full(len(points), np.inf, dtype=np.float64)
for start, end in zip(contour, np.roll(contour, -1, axis=0)):
segment = end - start
length_squared = float(segment.dot(segment))
if length_squared <= 1e-16:
continue
factor = np.clip(
(points - start) @ segment / length_squared,
0.0,
1.0,
)
nearest = start + factor[:, None] * segment
squared = np.minimum(
squared,
np.sum((points - nearest) ** 2, axis=1),
)
return np.sqrt(squared)
def chordal_axis(contour, epsilon=1e-9):
import numpy as np
from mathutils import Vector
from mathutils.geometry import delaunay_2d_cdt
contour = np.asarray(contour, dtype=np.float64)
boundary_count = len(contour)
boundary_edges = [
(index, (index + 1) % boundary_count) for index in range(boundary_count)
]
result = delaunay_2d_cdt(
[Vector(point) for point in contour],
boundary_edges,
[list(range(boundary_count))],
1,
epsilon,
)
points = np.asarray(
[tuple(point) for point in result[0]],
dtype=np.float64,
)
triangles = [tuple(face) for face in result[2] if len(face) == 3]
boundary = {tuple(sorted(edge)) for edge in boundary_edges}
node_indices = {}
nodes = []
segments = set()
def edge_node(edge):
key = ("edge", *tuple(sorted(edge)))
if key not in node_indices:
node_indices[key] = len(nodes)
nodes.append((points[edge[0]] + points[edge[1]]) * 0.5)
return node_indices[key]
for face_index, face in enumerate(triangles):
edges = [
(face[0], face[1]),
(face[1], face[2]),
(face[2], face[0]),
]
internal = [edge for edge in edges if tuple(sorted(edge)) not in boundary]
internal_nodes = [edge_node(edge) for edge in internal]
if len(internal_nodes) == 2:
segments.add(tuple(sorted(internal_nodes)))
elif len(internal_nodes) == 3:
key = ("face", face_index)
center_index = len(nodes)
node_indices[key] = center_index
nodes.append(points[list(face)].mean(axis=0))
for midpoint_index in internal_nodes:
segments.add(tuple(sorted((center_index, midpoint_index))))
if not nodes:
center = contour.mean(axis=0)
return (
np.asarray((center,), dtype=np.float64),
[],
np.asarray(
(distances_to_contour((center,), contour)[0],),
dtype=np.float64,
),
)
nodes = np.asarray(nodes, dtype=np.float64)
radii = distances_to_contour(nodes, contour)
active = set(segments)
while len(active) > 1:
degree = [0] * len(nodes)
for start, end in active:
degree[start] += 1
degree[end] += 1
removable = []
for edge in active:
start, end = edge
leaf = start if degree[start] == 1 else end if degree[end] == 1 else None
if leaf is None:
continue
length = float(np.linalg.norm(nodes[start] - nodes[end]))
if length < max(radii[leaf] * 0.5, epsilon):
removable.append(edge)
if not removable:
break
active.difference_update(removable)
if active:
used = sorted({index for edge in active for index in edge})
remap = {old: new for new, old in enumerate(used)}
nodes = nodes[used]
radii = radii[used]
active = {tuple(sorted((remap[start], remap[end]))) for start, end in active}
else:
widest = int(np.argmax(radii))
nodes = nodes[[widest]]
radii = radii[[widest]]
return nodes, sorted(active), radii
def smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
iterations=2,
factor=0.3,
):
import numpy as np
values = np.asarray(heights, dtype=np.float64).copy()
boundary_mask = np.asarray(boundary_mask, dtype=bool)
neighbors = [set() for _index in range(len(values))]
for face in planar_faces:
for start, end in zip(face, face[1:] + face[:1]):
neighbors[start].add(end)
neighbors[end].add(start)
for _iteration in range(iterations):
updated = values.copy()
for index in np.flatnonzero(~boundary_mask):
adjacent = [
neighbor for neighbor in neighbors[index] if not boundary_mask[neighbor]
]
if adjacent:
average = float(values[adjacent].mean())
updated[index] = values[index] * (1.0 - factor) + average * factor
updated[boundary_mask] = 0.0
values = updated
return values
def teddy_balloon_heights(
planar_points,
planar_faces,
contour,
boundary_mask,
inflation,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
boundary_mask = np.asarray(boundary_mask, dtype=bool)
nodes, segments, radii = chordal_axis(contour)
squared_heights = np.zeros(len(points), dtype=np.float64)
for start, end in segments:
axis_start = nodes[start]
axis_end = nodes[end]
axis = axis_end - axis_start
length_squared = float(axis.dot(axis))
if length_squared <= 1e-16:
continue
factors = np.clip(
(points - axis_start) @ axis / length_squared,
0.0,
1.0,
)
centers = axis_start + factors[:, None] * axis
local_radii = radii[start] + factors * (radii[end] - radii[start])
distance_squared = np.sum((points - centers) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
local_radii * local_radii - distance_squared,
)
degree = np.zeros(len(nodes), dtype=np.int64)
for start, end in segments:
degree[start] += 1
degree[end] += 1
for index in np.flatnonzero(degree != 2):
distance_squared = np.sum((points - nodes[index]) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
radii[index] * radii[index] - distance_squared,
)
missing = (~boundary_mask) & (squared_heights <= 0.0)
if np.any(missing):
fallback = distances_to_contour(points[missing], contour)
squared_heights[missing] = fallback * fallback
heights = inflation * np.sqrt(np.maximum(squared_heights, 0.0))
heights[boundary_mask] = 0.0
return smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
)
def cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
tolerance=1e-8,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
faces = np.asarray(planar_faces, dtype=np.int64)
if faces.size == 0:
faces = np.empty((0, 3), dtype=np.int64)
if faces.ndim != 2 or faces.shape[1] != 3:
raise ValueError("Balloon profile faces must be triangles")
boundary_mask = np.asarray(boundary_mask, dtype=bool)
triangles = points[faces]
a, b, c = (triangles[:, index] for index in range(3))
cross = np.abs(
(b[:, 0] - a[:, 0]) * (c[:, 1] - a[:, 1])
- (b[:, 1] - a[:, 1]) * (c[:, 0] - a[:, 0])
)
usable = cross > 1e-14
faces = faces[usable]
a, b, c = a[usable], b[usable], c[usable]
cross = cross[usable]
loads = np.bincount(
faces.ravel(),
weights=np.repeat(cross / 6.0, 3),
minlength=len(points),
)
edges = np.concatenate(
(
faces[:, (1, 2)],
faces[:, (0, 2)],
faces[:, (0, 1)],
),
axis=0,
)
edge_weights = np.concatenate(
(
np.sum((b - a) * (c - a), axis=1),
np.sum((a - b) * (c - b), axis=1),
np.sum((a - c) * (b - c), axis=1),
)
) / np.tile(cross * 2.0, 3)
edges.sort(axis=1)
edges, inverse = np.unique(edges, axis=0, return_inverse=True)
edge_weights = np.bincount(
inverse,
weights=edge_weights,
minlength=len(edges),
)
interior = np.flatnonzero(~boundary_mask)
field = np.zeros(len(points), dtype=np.float64)
if not len(interior):
return field
compact = np.full(len(points), -1, dtype=np.int64)
compact[interior] = np.arange(len(interior), dtype=np.int64)
# Constrained triangulation can create negative cotangent weights.
# Clamping them keeps the Dirichlet system positive definite.
edge_weights = np.maximum(edge_weights, 0.0)
usable = edge_weights > 1e-14
compact_edges = compact[edges[usable]]
edge_weights = edge_weights[usable]
start_compact = compact_edges[:, 0]
end_compact = compact_edges[:, 1]
start_usable = start_compact >= 0
end_usable = end_compact >= 0
diagonal = np.zeros(len(interior), dtype=np.float64)
diagonal += np.bincount(
start_compact[start_usable],
weights=edge_weights[start_usable],
minlength=len(interior),
)
diagonal += np.bincount(
end_compact[end_usable],
weights=edge_weights[end_usable],
minlength=len(interior),
)
if np.any(diagonal <= 1e-14):
raise ValueError("Unable to build a stable Balloon profile")
interior_edges = start_usable & end_usable
edge_starts = start_compact[interior_edges]
edge_ends = end_compact[interior_edges]
weights = edge_weights[interior_edges]
directed_starts = np.concatenate((edge_starts, edge_ends))
directed_ends = np.concatenate((edge_ends, edge_starts))
directed_weights = np.concatenate((weights, weights))
def multiply(values):
result = diagonal * values
if len(directed_weights):
result -= np.bincount(
directed_starts,
weights=directed_weights * values[directed_ends],
minlength=len(values),
)
return result
right_hand_side = loads[interior]
solution = np.zeros(len(interior), dtype=np.float64)
residual = right_hand_side.copy()
preconditioned = residual / diagonal
direction = preconditioned.copy()
residual_product = float(residual.dot(preconditioned))
target = max(float(np.linalg.norm(right_hand_side)) * tolerance, 1e-12)
max_iterations = max(64, min(len(interior) * 4, 4000))
for _iteration in range(max_iterations):
if float(np.linalg.norm(residual)) <= target:
break
multiplied = multiply(direction)
denominator = float(direction.dot(multiplied))
if denominator <= 0.0 or not math.isfinite(denominator):
raise ValueError("Unable to solve the Balloon profile")
step = residual_product / denominator
solution += step * direction
residual -= step * multiplied
preconditioned = residual / diagonal
next_product = float(residual.dot(preconditioned))
if not math.isfinite(next_product):
raise ValueError("Unable to solve the Balloon profile")
if abs(residual_product) <= 1e-30:
break
direction = preconditioned + next_product / residual_product * direction
residual_product = next_product
field[interior] = np.maximum(solution, 0.0)
return field
def poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation=DEFAULT_INFLATION,
):
import numpy as np
field = cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
)
return 2.0 * float(inflation) * np.sqrt(field)
def balloon_profile(
algorithm,
planar_points,
planar_faces,
boundary_mask,
components,
inflation=DEFAULT_INFLATION,
):
import numpy as np
algorithm = str(algorithm).upper()
if algorithm == "POISSON":
return poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation,
)
if algorithm != "TEDDY":
raise ValueError(f"Unknown Balloon profile algorithm: {algorithm}")
result = np.zeros(len(planar_points), dtype=np.float64)
for start, end, component_faces, outer_contour in components:
result[start:end] = teddy_balloon_heights(
np.asarray(planar_points)[start:end],
component_faces,
outer_contour,
np.asarray(boundary_mask)[start:end],
inflation,
)
return result
PLANE_VARIANCE_FLOOR_RATIO = 0.025
DEPTH_REFERENCE_PERCENTILE = 95.0
ADAPTIVE_DEPTH_TRANSPARENCY_RATIO = 0.05
DEFAULT_DEPTH_BASELINE = 8.0
DEFAULT_DEPTH_DIRECTION = (0.0, 0.0, 1.0)
def use_adaptive_depth_calibration(selection_transparent_fraction):
"""Return whether the Selection input has enough transparent context."""
return float(selection_transparent_fraction) >= ADAPTIVE_DEPTH_TRANSPARENCY_RATIO
def metric_depth_reference(camera_depth, valid, uniform_scale):
"""Return the shared upper-percentile scene depth for both Depth shapes."""
import numpy as np
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth values and validity mask dimensions do not match")
usable = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
if not usable.any():
raise ValueError("The depth field contains no valid metric depth")
if not uniform_scale > 0.0:
raise ValueError("Depth Uniform Scale must be positive")
reference = np.percentile(
camera_depth[usable],
DEPTH_REFERENCE_PERCENTILE,
)
return float(reference) * float(uniform_scale)
def _sample_valid_bilinear(values, valid, x, y):
import numpy as np
height, width = values.shape
x = np.clip(x, 0.0, max(width - 1, 0))
y = np.clip(y, 0.0, max(height - 1, 0))
x0 = np.floor(x).astype(np.int64)
y0 = np.floor(y).astype(np.int64)
x1 = np.minimum(x0 + 1, width - 1)
y1 = np.minimum(y0 + 1, height - 1)
fx = x - x0
fy = y - y0
samples = np.stack(
(
values[y0, x0],
values[y0, x1],
values[y1, x0],
values[y1, x1],
),
axis=1,
)
weights = np.stack(
(
(1.0 - fx) * (1.0 - fy),
fx * (1.0 - fy),
(1.0 - fx) * fy,
fx * fy,
),
axis=1,
)
sample_valid = np.stack(
(
valid[y0, x0],
valid[y0, x1],
valid[y1, x0],
valid[y1, x1],
),
axis=1,
)
weights *= sample_valid
totals = weights.sum(axis=1)
result = np.zeros(len(x), dtype=np.float32)
usable = totals > 1e-8
result[usable] = (samples[usable] * weights[usable]).sum(axis=1) / totals[usable]
return result, usable
def _weighted_plane(coordinates, values, weights):
import numpy as np
total = float(np.sum(weights))
if total <= 1e-12:
return np.zeros(3, dtype=np.float32)
center = np.sum(coordinates * weights[:, None], axis=0) / total
value_center = float(np.sum(values * weights) / total)
centered = coordinates - center
covariance = (centered * weights[:, None]).T @ centered / total
cross = centered.T @ (weights * (values - value_center)) / total
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
largest = max(float(eigenvalues[-1]), 0.0)
if largest <= 1e-12:
slopes = np.zeros(2, dtype=np.float64)
else:
supported = np.maximum(eigenvalues, largest * PLANE_VARIANCE_FLOOR_RATIO)
slopes = eigenvectors @ ((eigenvectors.T @ cross) / supported)
intercept = value_center - float(center @ slopes)
return np.asarray((*slopes, intercept), dtype=np.float32)
def _stable_plane(coordinates, values):
import numpy as np
coordinates = np.asarray(coordinates, dtype=np.float64)
values = np.asarray(values, dtype=np.float64)
if len(coordinates) < 2:
return np.asarray((0.0, 0.0, np.median(values)), dtype=np.float32)
weights = np.ones(len(values), dtype=np.float64)
plane = np.zeros(3, dtype=np.float32)
for _iteration in range(4):
plane = _weighted_plane(coordinates, values, weights)
residual = values - coordinates @ plane[:2] - plane[2]
centered = residual - np.median(residual)
scale = max(float(np.median(np.abs(centered))) * 1.4826, 1e-6)
weights = np.minimum(
1.0,
(2.5 * scale) / np.maximum(np.abs(centered), 1e-12),
)
return plane
def depth_balloon_projection_parameters(
vertices,
vertex_uv,
camera_depth,
valid,
uniform_scale,
):
"""Fit a metric base plane and return its center depth and direction."""
import numpy as np
vertices = np.asarray(vertices, dtype=np.float32)
uv = np.asarray(vertex_uv, dtype=np.float32)
if vertices.ndim != 2 or vertices.shape[1] != 3:
raise ValueError("Depth Balloon vertices must have three components")
if uv.shape != (len(vertices), 2):
raise ValueError("Depth Balloon UV coordinates do not match its vertices")
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth Balloon depth and mask dimensions do not match")
if not uniform_scale > 0.0:
raise ValueError("Depth Balloon Uniform Scale must be positive")
height, width = camera_depth.shape
sample_x = uv[:, 0] * max(width - 1, 0)
sample_y = (1.0 - uv[:, 1]) * max(height - 1, 0)
calibration_valid = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
sampled, usable = _sample_valid_bilinear(
camera_depth,
calibration_valid,
sample_x,
sample_y,
)
if not usable.any():
raise ValueError("MoGe-2 found no valid Depth Balloon samples")
local_coordinates = vertices[:, :2]
metric_depth = sampled[usable] * float(uniform_scale)
plane = _stable_plane(local_coordinates[usable], metric_depth)
reference = metric_depth_reference(
camera_depth,
calibration_valid,
uniform_scale,
)
direction = np.asarray((plane[0], plane[1], 1.0), dtype=np.float64)
direction /= np.linalg.norm(direction)
return reference, tuple(float(value) for value in direction)
CUTOUT_ALPHA_THRESHOLD = 0.9
CUTOUT_CLEANUP_UI_SCALE = 100.0
@@ -1313,7 +752,7 @@ def cutout_domain(
planar = np.concatenate(refined_points)
planar_faces, components = refined_faces, refined_components
boundary = boundary_vertices_from_faces(len(planar), planar_faces)
profile = balloon_profile(
profile = balloon.balloon_profile(
balloon_algorithm, planar, planar_faces, boundary, components
)
faces = np.asarray(planar_faces, dtype=np.int32)