icrdr/AnyImage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor depth reference handling and cleanup

Browse Source 
- Unified reference depth calculation to use a single implementation in common/depth.py.
- Removed server-side reference depth computation and adjusted depth.json to only include image_size and intrinsics.
- Updated depth texture handling to ignore transparent, invalid, and non-positive pixels when calculating reference depth.
- Changed reference depth calculation to use the 95th percentile of valid depth values.
- Adjusted depth direction fitting to return a flat direction when no samples are available.
- Removed cleanup threshold parameters from Cutout and related nodes, simplifying the interface.
- Updated tests to verify new reference depth logic and ensure compatibility with existing functionality.
This commit is contained in:
马南
2026-09-15 22:50:16 +08:00
parent 94130ff38f
commit 2dfa9b2add
37 changed files with 665 additions and 639 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
src/anyimage/assets/O_AnyImage.blend LFS
View File

Binary file not shown.

125
src/anyimage/common/depth.py
View File

@@ -9,6 +9,10 @@ from .image import depth_data_name, image_empty_bounds
PLANE_VARIANCE_FLOOR_RATIO = 0.025
REFERENCE_DEPTH_PERCENTILE = 95.0
DEPTH_ALPHA_THRESHOLD = 0.95
REFERENCE_DEPTH_LIMIT = 8.0
FLAT_DEPTH_DIRECTION = (0.0, 1.0, 0.0)
def load_depth_metadata(path):
@@ -20,7 +24,6 @@ def load_depth_metadata(path):
tuple(float(value) for value in row)
for row in metadata["intrinsics"]
)
reference_depth = float(metadata["reference_depth"])
except (KeyError, TypeError, ValueError) as error:
raise ValueError("Depth metadata is invalid") from error
if (
@@ -30,13 +33,11 @@ def load_depth_metadata(path):
or any(len(row) != 3 for row in intrinsics)
or intrinsics[0][0] <= 0.0
or intrinsics[1][1] <= 0.0
or reference_depth <= 0.0
):
raise ValueError("Depth metadata is invalid")
return {
"image_size": image_size,
"intrinsics": intrinsics,
"reference_depth": reference_depth,
}
@@ -64,7 +65,32 @@ def camera_depth_values(image):
pixels = np.empty(width * height * 4, dtype=np.float32)
image.pixels.foreach_get(pixels)
pixels = np.flipud(pixels.reshape((height, width, 4)))
return pixels[..., 2], pixels[..., 3] > 0.5
return pixels[..., 2], pixels[..., 3] > DEPTH_ALPHA_THRESHOLD
def reference_depth(image, mask=None):
"""Return the upper-percentile camera depth of a loaded Depth texture.
Distances are in model meters. Samples farther than
``REFERENCE_DEPTH_LIMIT`` are dropped so a distant background cannot push
the reference plane, and with it the whole reconstruction scale, out to
the far field. The limit itself is the reference when the selection leaves
no sample.
"""
import numpy as np
depth, valid = camera_depth_values(image)
usable = valid & np.isfinite(depth) & (depth > 0.0)
if mask is not None:
mask = np.asarray(mask, dtype=bool)
if mask.shape != depth.shape:
raise ValueError("Depth reference mask dimensions do not match")
usable &= mask
values = depth[usable]
values = values[values <= REFERENCE_DEPTH_LIMIT]
if not values.size:
return REFERENCE_DEPTH_LIMIT
return float(np.percentile(values, REFERENCE_DEPTH_PERCENTILE))
def _weighted_depth_plane(coordinates, values, weights):
@@ -90,7 +116,11 @@ def _weighted_depth_plane(coordinates, values, weights):
def fit_depth_direction(coordinates, depth, valid, uniform_scale):
"""Fit a robust metric depth plane on the XZ plane and return its direction."""
"""Fit a robust metric depth plane on the XZ plane and return its direction.
The direction is canonical `(slope_x, 1, slope_z)`. A selection without
usable samples returns `FLAT_DEPTH_DIRECTION`.
"""
import numpy as np
coordinates = np.asarray(coordinates, dtype=np.float64)
@@ -103,10 +133,10 @@ def fit_depth_direction(coordinates, depth, valid, uniform_scale):
if not uniform_scale > 0.0:
raise ValueError("Depth Uniform Scale must be positive")
usable = valid & np.isfinite(depth) & (depth > 0.0)
if not usable.any():
raise ValueError("The depth field contains no valid metric depth")
values = depth[usable] * float(uniform_scale)
points = coordinates[usable]
if not len(points):
return FLAT_DEPTH_DIRECTION
if len(points) < 2:
plane = np.asarray((0.0, 0.0, np.median(values)), dtype=np.float32)
else:
@@ -126,6 +156,80 @@ def fit_depth_direction(coordinates, depth, valid, uniform_scale):
return tuple(float(value) for value in direction)
def _sample_valid_bilinear(values, valid, x, y):
import numpy as np
height, width = values.shape
x = np.clip(x, 0.0, max(width - 1, 0))
y = np.clip(y, 0.0, max(height - 1, 0))
x0 = np.floor(x).astype(np.int64)
y0 = np.floor(y).astype(np.int64)
x1 = np.minimum(x0 + 1, width - 1)
y1 = np.minimum(y0 + 1, height - 1)
fx = x - x0
fy = y - y0
samples = np.stack(
(
values[y0, x0],
values[y0, x1],
values[y1, x0],
values[y1, x1],
),
axis=1,
)
weights = np.stack(
(
(1.0 - fx) * (1.0 - fy),
fx * (1.0 - fy),
(1.0 - fx) * fy,
fx * fy,
),
axis=1,
)
sample_valid = np.stack(
(
valid[y0, x0],
valid[y0, x1],
valid[y1, x0],
valid[y1, x1],
),
axis=1,
)
weights *= sample_valid
totals = weights.sum(axis=1)
result = np.zeros(len(x), dtype=np.float32)
usable = totals > 1e-8
result[usable] = (samples[usable] * weights[usable]).sum(axis=1) / totals[usable]
return result, usable
def fit_symmetry_depth_direction(image, vertices, vertex_uv, uniform_scale):
"""Fit the Depth Symmetry base plane direction from base shape UV samples."""
import numpy as np
vertices = np.asarray(vertices, dtype=np.float32)
uv = np.asarray(vertex_uv, dtype=np.float32)
if vertices.ndim != 2 or vertices.shape[1] != 3:
raise ValueError("Depth Symmetry vertices must have three components")
if uv.shape != (len(vertices), 2):
raise ValueError("Depth Symmetry UV coordinates do not match its vertices")
depth, valid = camera_depth_values(image)
height, width = depth.shape
usable = valid & np.isfinite(depth) & (depth > 0.0)
sampled, sampled_usable = _sample_valid_bilinear(
depth,
usable,
uv[:, 0] * max(width - 1, 0),
(1.0 - uv[:, 1]) * max(height - 1, 0),
)
return fit_depth_direction(
vertices[:, (0, 2)],
sampled,
sampled_usable,
uniform_scale,
)
def fit_image_depth_direction(image, bounds, uniform_scale):
"""Fit the depth direction across a complete XZ image plane."""
import numpy as np
@@ -145,17 +249,16 @@ def fit_image_depth_direction(image, bounds, uniform_scale):
)
def depth_uniform_scale(source_object, metadata, bounds=None):
def depth_uniform_scale(source_object, metadata, model_reference, bounds=None):
"""Convert model camera units uniformly into Image Empty local distance."""
import numpy as np
intrinsics = metadata["intrinsics"]
width, height = metadata["image_size"]
reference_depth = metadata["reference_depth"]
x_min, x_max, y_min, y_max = bounds or image_empty_bounds(source_object)
plane_width = x_max - x_min
plane_height = y_max - y_min
fx, fy = float(intrinsics[0][0]), float(intrinsics[1][1])
scale_x = plane_width * fx / (max(width - 1, 1) * reference_depth)
scale_y = plane_height * fy / (max(height - 1, 1) * reference_depth)
scale_x = plane_width * fx / (max(width - 1, 1) * model_reference)
scale_y = plane_height * fy / (max(height - 1, 1) * model_reference)
return float(np.sqrt(scale_x * scale_y))

18
src/anyimage/operators/convert_to_plane/object.py
View File

@@ -12,6 +12,7 @@ from ...common.depth import (
fit_image_depth_direction,
load_depth_metadata,
load_depth_result_image,
reference_depth,
)
from ...common.coordinate import canonical_direction_to_legacy
from ...common.material import create_image_material
@@ -81,23 +82,26 @@ def create_depth_plane_object(
modifier = plane.modifiers.new(name=node_group.name, type="NODES")
modifier.node_group = node_group
modifier.show_group_selector = False
uniform_scale = depth_uniform_scale(source_object, depth_metadata)
model_reference = reference_depth(depth_image)
uniform_scale = depth_uniform_scale(
source_object,
depth_metadata,
model_reference,
)
inputs = [
("Subdivide", max(0, min(10, int(subdivisions)))),
("Reference Depth", depth_metadata["reference_depth"] * uniform_scale),
("Reference Depth", model_reference * uniform_scale),
("Uniform Scale", uniform_scale),
("Depth Image", depth_image),
]
if plane_type == "RELIEF":
try:
depth_direction = fit_image_depth_direction(
depth_direction = canonical_direction_to_legacy(
fit_image_depth_direction(
depth_image,
bounds,
uniform_scale,
)
except ValueError:
depth_direction = (0.0, 0.0, 1.0)
depth_direction = canonical_direction_to_legacy(depth_direction)
)
inputs.append((
"Depth Direction",
depth_direction,

131
src/anyimage/operators/cutout_tool/depth_calibration.py
View File

@@ -1,131 +0,0 @@
"""Calibrate metric reference depth and the Depth Symmetry base plane."""
from ...common.depth import fit_depth_direction
from ...common.coordinate import canonical_direction_to_symmetry_legacy
DEPTH_REFERENCE_PERCENTILE = 95.0
ADAPTIVE_DEPTH_TRANSPARENCY_RATIO = 0.05
DEFAULT_DEPTH_BASELINE = 8.0
DEFAULT_DEPTH_DIRECTION = (0.0, 0.0, 1.0)
def use_adaptive_depth_calibration(selection_transparent_fraction):
"""Return whether the Selection input has enough transparent context."""
return float(selection_transparent_fraction) >= ADAPTIVE_DEPTH_TRANSPARENCY_RATIO
def metric_depth_reference(camera_depth, valid, uniform_scale):
"""Return the shared upper-percentile scene depth for both Depth shapes."""
import numpy as np
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth values and validity mask dimensions do not match")
usable = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
if not usable.any():
raise ValueError("The depth field contains no valid metric depth")
if not uniform_scale > 0.0:
raise ValueError("Depth Uniform Scale must be positive")
reference = np.percentile(
camera_depth[usable],
DEPTH_REFERENCE_PERCENTILE,
)
return float(reference) * float(uniform_scale)
def _sample_valid_bilinear(values, valid, x, y):
import numpy as np
height, width = values.shape
x = np.clip(x, 0.0, max(width - 1, 0))
y = np.clip(y, 0.0, max(height - 1, 0))
x0 = np.floor(x).astype(np.int64)
y0 = np.floor(y).astype(np.int64)
x1 = np.minimum(x0 + 1, width - 1)
y1 = np.minimum(y0 + 1, height - 1)
fx = x - x0
fy = y - y0
samples = np.stack(
(
values[y0, x0],
values[y0, x1],
values[y1, x0],
values[y1, x1],
),
axis=1,
)
weights = np.stack(
(
(1.0 - fx) * (1.0 - fy),
fx * (1.0 - fy),
(1.0 - fx) * fy,
fx * fy,
),
axis=1,
)
sample_valid = np.stack(
(
valid[y0, x0],
valid[y0, x1],
valid[y1, x0],
valid[y1, x1],
),
axis=1,
)
weights *= sample_valid
totals = weights.sum(axis=1)
result = np.zeros(len(x), dtype=np.float32)
usable = totals > 1e-8
result[usable] = (samples[usable] * weights[usable]).sum(axis=1) / totals[usable]
return result, usable
def depth_symmetry_projection_parameters(
vertices,
vertex_uv,
camera_depth,
valid,
uniform_scale,
):
"""Fit a metric base plane and return its center depth and direction."""
import numpy as np
vertices = np.asarray(vertices, dtype=np.float32)
uv = np.asarray(vertex_uv, dtype=np.float32)
if vertices.ndim != 2 or vertices.shape[1] != 3:
raise ValueError("Depth Symmetry vertices must have three components")
if uv.shape != (len(vertices), 2):
raise ValueError("Depth Symmetry UV coordinates do not match its vertices")
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth Symmetry depth and mask dimensions do not match")
if not uniform_scale > 0.0:
raise ValueError("Depth Symmetry Uniform Scale must be positive")
height, width = camera_depth.shape
sample_x = uv[:, 0] * max(width - 1, 0)
sample_y = (1.0 - uv[:, 1]) * max(height - 1, 0)
calibration_valid = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
sampled, usable = _sample_valid_bilinear(
camera_depth,
calibration_valid,
sample_x,
sample_y,
)
if not usable.any():
raise ValueError("MoGe-2 found no valid Depth Symmetry samples")
local_coordinates = vertices[:, (0, 2)]
reference = metric_depth_reference(
camera_depth,
calibration_valid,
uniform_scale,
)
direction = fit_depth_direction(
local_coordinates,
sampled,
usable,
uniform_scale,
)
return reference, canonical_direction_to_symmetry_legacy(direction)

34
src/anyimage/operators/cutout_tool/geometry.py
View File

@@ -464,7 +464,6 @@ def boundary_vertices_from_faces(vertex_count, faces):
CUTOUT_ALPHA_THRESHOLD = 0.9
CUTOUT_CLEANUP_UI_SCALE = 100.0
@dataclass(frozen=True)
@@ -532,17 +531,6 @@ def world_plane_metric(matrix_world):
return ((x.dot(x), x.dot(y)), (x.dot(y), y.dot(y))), z.length
def cutout_cleanup_value(spacing, world_metric):
import numpy as np
if spacing <= 0.0:
raise ValueError("Cutout point spacing must be greater than zero")
area_scale = abs(float(np.linalg.det(metric_transform(world_metric))))
if area_scale <= 1e-12:
raise ValueError("The Image Empty has a degenerate world transform")
return CUTOUT_CLEANUP_UI_SCALE * float(spacing) ** 2 / area_scale
def counter_clockwise_faces(points, faces):
import numpy as np
@@ -620,27 +608,13 @@ def effective_cutout_edge_length(
)
def selection_calibration(
content_values, target_shape, threshold=CUTOUT_ALPHA_THRESHOLD
):
"""Return Cutout validity and transparency ratio at geometry resolution."""
def build_reference_mask(content_values, image_size):
"""Resample Cutout content into a boolean mask at Depth texture resolution."""
import numpy as np
values = np.asarray(content_values, dtype=np.float32)
target_height, target_width = (int(value) for value in target_shape)
source_height, source_width = values.shape
y = np.minimum(
(np.arange(target_height) * source_height / target_height).astype(np.int64),
source_height - 1,
)
x = np.minimum(
(np.arange(target_width) * source_width / target_width).astype(np.int64),
source_width - 1,
)
resized = values[y[:, None], x[None, :]]
valid = (resized > 0.0) & (resized >= float(threshold))
transparent_fraction = float(np.count_nonzero(~valid) / valid.size)
return valid, transparent_fraction
target_width, target_height = (int(value) for value in image_size)
return _resize_values(values, (target_width, target_height)) > 0.5
def build_base_shape(

77
src/anyimage/operators/cutout_tool/object.py
View File

@@ -4,7 +4,12 @@ from enum import StrEnum
import bpy
from ...common.depth import camera_depth_values, depth_uniform_scale
from ...common.coordinate import canonical_direction_to_symmetry_legacy
from ...common.depth import (
depth_uniform_scale,
fit_symmetry_depth_direction,
reference_depth,
)
from ...common.material import create_image_material
from ...common.object import (
blender_frame_rotation,
@@ -13,19 +18,9 @@ from ...common.object import (
set_modifier_input,
)
from ...common.node import load_node_group
from .depth_calibration import (
DEFAULT_DEPTH_BASELINE,
DEFAULT_DEPTH_DIRECTION,
depth_symmetry_projection_parameters,
metric_depth_reference,
use_adaptive_depth_calibration,
)
from .geometry import (
CUTOUT_ALPHA_THRESHOLD,
build_reference_mask,
crop_plane_bounds,
cutout_cleanup_value,
selection_calibration,
world_plane_metric,
)
from .shape import (
BALLOON_ATTRIBUTE_NAME,
@@ -80,12 +75,10 @@ def create_shape_object(
context,
source_object,
shape,
edge_length,
content_values,
content_bounds,
base_shape,
*,
alpha_threshold=CUTOUT_ALPHA_THRESHOLD,
depth_image=None,
depth_metadata=None,
normal_image=None,
@@ -106,11 +99,6 @@ def create_shape_object(
(bounds[0] + bounds[1]) * 0.5,
(bounds[2] + bounds[3]) * 0.5,
)
world_metric, _world_z_scale = world_plane_metric(source_object.matrix_world)
cleanup_value = cutout_cleanup_value(
edge_length,
world_metric,
)
material = None
mesh = None
mesh_object = None
@@ -132,33 +120,25 @@ def create_shape_object(
if shape in DEPTH_CUTOUT_SHAPES:
if depth_image is None or depth_metadata is None:
raise ValueError("Depth Shape requires a Depth texture and metadata")
camera_depth, valid = camera_depth_values(depth_image)
calibration_valid, transparent_fraction = selection_calibration(
content_values, valid.shape, threshold=alpha_threshold
model_reference = reference_depth(
depth_image,
build_reference_mask(content_values, depth_image.size),
)
calibration_valid &= valid
uniform_scale = depth_uniform_scale(
source_object,
depth_metadata,
model_reference,
bounds,
)
if not use_adaptive_depth_calibration(transparent_fraction):
depth_reference = DEFAULT_DEPTH_BASELINE
if shape == "DEPTH_SYMMETRY":
depth_direction = DEFAULT_DEPTH_DIRECTION
elif shape == "DEPTH_SOLID":
depth_reference = metric_depth_reference(
camera_depth,
calibration_valid,
uniform_scale,
)
else:
depth_reference, depth_direction = depth_symmetry_projection_parameters(
base_shape.vertices,
base_shape.uv,
camera_depth,
calibration_valid,
uniform_scale,
depth_reference = model_reference * uniform_scale
if shape == "DEPTH_SYMMETRY":
depth_direction = canonical_direction_to_symmetry_legacy(
fit_symmetry_depth_direction(
depth_image,
base_shape.vertices,
base_shape.uv,
uniform_scale,
)
)
source_name = source_object.name
@@ -195,22 +175,15 @@ def create_shape_object(
modifier = mesh_object.modifiers.new(name=node_group.name, type="NODES")
modifier.node_group = node_group
modifier.show_group_selector = False
for name, value in (
("Cleanup Threshold", cleanup_value),
(
"Thickness",
0.0 if shape in {"FLAT", "DEPTH_SYMMETRY"} else 1.0,
),
):
if name == "Thickness" and shape == "DEPTH_SYMMETRY":
continue
if shape != "DEPTH_SYMMETRY":
set_modifier_input(
modifier,
modifier_input_identifier(
node_group, name,
subtype="NONE" if name == "Thickness" and shape in DEPTH_CUTOUT_SHAPES else None,
node_group,
"Thickness",
subtype="NONE" if shape in DEPTH_CUTOUT_SHAPES else None,
),
value,
0.0 if shape == "FLAT" else 1.0,
)
if shape == "DEPTH_SOLID":
set_modifier_input(

3
src/anyimage/operators/cutout_tool/operators.py
View File

@@ -138,11 +138,9 @@ def create_cutout_shape(
context,
source_object,
shape,
effective_length,
content_values,
content_bounds,
base_shape,
alpha_threshold=alpha_threshold,
depth_image=depth_image,
depth_metadata=depth_metadata,
normal_image=normal_image,
@@ -320,7 +318,6 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
"generate_depth": generate_depth,
"normal_mode": normal_mode.value,
"max_input_size": int(self.max_ai_input_size),
"alpha_threshold": float(self.alpha_threshold),
}
if generate_depth or normal_mode != NormalMode.NONE:
parameters.update(

9
src/anyimage/server/geometry/depth_texture.py
View File

@@ -90,30 +90,23 @@ def write_depth_texture(frame, output_path, *, alpha):
return write_float_exr(rgba, output_path)
def write_depth_metadata(frame, reference_valid, output_path):
def write_depth_metadata(frame, output_path):
"""Write camera values needed to scale a model-space depth texture."""
import numpy as np
depth = np.asarray(frame.depth, dtype=np.float32)
validity = np.asarray(frame.validity, dtype=np.float32)
intrinsics = np.asarray(frame.intrinsics, dtype=np.float32)
reference_valid = np.asarray(reference_valid, dtype=bool)
if depth.ndim != 2 or validity.shape != depth.shape:
raise ValueError("Depth metadata dimensions do not match")
if reference_valid.shape != depth.shape:
raise ValueError("Depth metadata reference mask dimensions do not match")
if intrinsics.shape != (3, 3):
raise ValueError("Depth metadata requires 3x3 camera intrinsics")
valid = (validity > 0.5) & np.isfinite(depth) & (depth > 0.0)
reference_valid &= valid
if not valid.any():
raise ValueError("Geometry frame contains no valid depth pixels")
if not reference_valid.any():
raise ValueError("The image contains no valid depth pixels")
metadata = {
"image_size": [int(depth.shape[1]), int(depth.shape[0])],
"intrinsics": intrinsics.tolist(),
"reference_depth": float(np.median(depth[reference_valid])),
}
output_path = Path(output_path)
output_path.write_text(

2
src/anyimage/server/geometry/prediction_artifacts.py
View File

@@ -52,7 +52,6 @@ def generate_moge_artifacts(
*,
generate_depth=False,
normal_mode="NONE",
alpha_threshold=0.9,
):
"""Run MoGe once and write only the requested file artifacts."""
normal_mode = str(normal_mode).upper()
@@ -80,7 +79,6 @@ def generate_moge_artifacts(
)
result["depth_metadata"] = write_depth_metadata(
frame,
(alpha > 0.0) & (alpha >= alpha_threshold),
context.directory / "depth.json",
)

1
src/anyimage/server/jobs/cutout.py
View File

@@ -19,7 +19,6 @@ def run(context, parameters):
input_path,
generate_depth=generate_depth,
normal_mode=normal_mode,
alpha_threshold=float(parameters.get("alpha_threshold", 0.9)),
)
result.update({key: path.name for key, path in artifacts.items()})
context.check_cancelled()