feat(cutout): implement boundary padding for cutout textures
- Introduced a new `Boundary Padding` preference in the Cutout Tool with a default of 2 px and a configurable range of 1-4 px. - Updated the Cutout client to establish inward mapping from the final Mask, sampling in a fixed direction and processing independent material RGBA and Depth. - Enhanced the handling of RGBA and Depth during cutout operations, ensuring proper mapping and preservation of source data. - Added tests to validate the new boundary padding functionality and its integration with existing cutout processes. feat(geometry): remove overall mesh smoothing from geometry nodes - Removed the `Smooth` input from several geometry node groups to eliminate unnecessary overall mesh smoothing. - Retained `Smooth Weight` for depth nodes while removing it from Cutout and Relief Plane nodes. - Ensured that boundary, normal, and fill smoothing behaviors remain intact and consistent. - Updated tests and documentation to reflect the changes in node interfaces and behaviors. feat(normals): scale tangent normals with depth - Reused the existing `o_normal_reduction` attribute to adjust Tangent Normal strength based on the geometry's `Depth Scale`. - Ensured that Tangent Normal strength is consistent with the geometry depth, preserving full strength for materials that do not provide the attribute. - Updated tests to verify the correct behavior of normal strength adjustments in both Tangent and Object Space.
This commit is contained in:
Binary file not shown.
@@ -8,6 +8,9 @@ import bpy
|
||||
from .image import depth_data_name, image_empty_bounds
|
||||
|
||||
|
||||
PLANE_VARIANCE_FLOOR_RATIO = 0.025
|
||||
|
||||
|
||||
def load_depth_metadata(path):
|
||||
"""Read and validate the compact camera metadata beside a Depth texture."""
|
||||
metadata = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
@@ -64,6 +67,84 @@ def camera_depth_values(image):
|
||||
return pixels[..., 2], pixels[..., 3] > 0.5
|
||||
|
||||
|
||||
def _weighted_depth_plane(coordinates, values, weights):
|
||||
import numpy as np
|
||||
|
||||
total = float(np.sum(weights))
|
||||
if total <= 1e-12:
|
||||
return np.zeros(3, dtype=np.float32)
|
||||
center = np.sum(coordinates * weights[:, None], axis=0) / total
|
||||
value_center = float(np.sum(values * weights) / total)
|
||||
centered = coordinates - center
|
||||
covariance = (centered * weights[:, None]).T @ centered / total
|
||||
cross = centered.T @ (weights * (values - value_center)) / total
|
||||
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
|
||||
largest = max(float(eigenvalues[-1]), 0.0)
|
||||
if largest <= 1e-12:
|
||||
slopes = np.zeros(2, dtype=np.float64)
|
||||
else:
|
||||
supported = np.maximum(eigenvalues, largest * PLANE_VARIANCE_FLOOR_RATIO)
|
||||
slopes = eigenvectors @ ((eigenvectors.T @ cross) / supported)
|
||||
intercept = value_center - float(center @ slopes)
|
||||
return np.asarray((*slopes, intercept), dtype=np.float32)
|
||||
|
||||
|
||||
def fit_depth_direction(coordinates, depth, valid, uniform_scale):
|
||||
"""Fit a robust metric depth plane and return its normalized direction."""
|
||||
import numpy as np
|
||||
|
||||
coordinates = np.asarray(coordinates, dtype=np.float64)
|
||||
depth = np.asarray(depth, dtype=np.float64)
|
||||
valid = np.asarray(valid, dtype=bool)
|
||||
if coordinates.ndim != 2 or coordinates.shape[1] != 2:
|
||||
raise ValueError("Depth plane coordinates must have two components")
|
||||
if depth.shape != (len(coordinates),) or valid.shape != depth.shape:
|
||||
raise ValueError("Depth plane samples and validity must match coordinates")
|
||||
if not uniform_scale > 0.0:
|
||||
raise ValueError("Depth Uniform Scale must be positive")
|
||||
usable = valid & np.isfinite(depth) & (depth > 0.0)
|
||||
if not usable.any():
|
||||
raise ValueError("The depth field contains no valid metric depth")
|
||||
values = depth[usable] * float(uniform_scale)
|
||||
points = coordinates[usable]
|
||||
if len(points) < 2:
|
||||
plane = np.asarray((0.0, 0.0, np.median(values)), dtype=np.float32)
|
||||
else:
|
||||
weights = np.ones(len(values), dtype=np.float64)
|
||||
plane = np.zeros(3, dtype=np.float32)
|
||||
for _iteration in range(4):
|
||||
plane = _weighted_depth_plane(points, values, weights)
|
||||
residual = values - points @ plane[:2] - plane[2]
|
||||
centered = residual - np.median(residual)
|
||||
scale = max(float(np.median(np.abs(centered))) * 1.4826, 1e-6)
|
||||
weights = np.minimum(
|
||||
1.0,
|
||||
(2.5 * scale) / np.maximum(np.abs(centered), 1e-12),
|
||||
)
|
||||
direction = np.asarray((plane[0], plane[1], 1.0), dtype=np.float64)
|
||||
direction /= np.linalg.norm(direction)
|
||||
return tuple(float(value) for value in direction)
|
||||
|
||||
|
||||
def fit_image_depth_direction(image, bounds, uniform_scale):
|
||||
"""Fit the depth direction across a complete image plane."""
|
||||
import numpy as np
|
||||
|
||||
depth, valid = camera_depth_values(image)
|
||||
height, width = depth.shape
|
||||
x_min, x_max, y_min, y_max = bounds
|
||||
x = x_min + (np.arange(width) + 0.5) / width * (x_max - x_min)
|
||||
y = y_max - (np.arange(height) + 0.5) / height * (y_max - y_min)
|
||||
xx, yy = np.meshgrid(x, y)
|
||||
coordinates = np.column_stack((xx.ravel(), yy.ravel()))
|
||||
return fit_depth_direction(
|
||||
coordinates,
|
||||
depth.ravel(),
|
||||
valid.ravel(),
|
||||
uniform_scale,
|
||||
)
|
||||
|
||||
|
||||
def depth_uniform_scale(source_object, metadata, bounds=None):
|
||||
"""Convert model camera units uniformly into Image Empty local distance."""
|
||||
import numpy as np
|
||||
|
||||
@@ -84,6 +84,18 @@ def image_rgba(image):
|
||||
return np.flipud(image_pixels(image).reshape((height, width, 4)))
|
||||
|
||||
|
||||
def panorama_crop_bounds(width, height):
|
||||
"""Return the largest centered integer-pixel 2:1 bounds."""
|
||||
width, height = int(width), int(height)
|
||||
crop_height = min(height, width // 2)
|
||||
crop_width = crop_height * 2
|
||||
if crop_width <= 0 or crop_height <= 0:
|
||||
raise ValueError("Panorama conversion requires an image at least 2 x 1 pixels")
|
||||
left = (width - crop_width) // 2
|
||||
top = (height - crop_height) // 2
|
||||
return left, top, left + crop_width, top + crop_height
|
||||
|
||||
|
||||
def homography_from_points(source, target):
|
||||
"""Return a projective transform mapping four source points to target."""
|
||||
import numpy as np
|
||||
|
||||
@@ -108,39 +108,20 @@ def create_image_material(
|
||||
nodes.clear()
|
||||
|
||||
color_texture = nodes.new("ShaderNodeTexImage")
|
||||
uv_map = nodes.new("ShaderNodeUVMap")
|
||||
output = nodes.new("ShaderNodeOutputMaterial")
|
||||
|
||||
uv_map.uv_map = "UVMap"
|
||||
uv_map.location = (-1250.0, 350.0)
|
||||
color_texture.image = material_image
|
||||
color_texture.interpolation = "Linear"
|
||||
color_texture.extension = texture_extension
|
||||
color_texture.location = (-1000.0, 350.0)
|
||||
output.location = (850.0, 350.0)
|
||||
color_texture.location = (-700.0, 300.0)
|
||||
output.location = (500.0, 300.0)
|
||||
links = material.node_tree.links
|
||||
links.new(uv_map.outputs["UV"], color_texture.inputs["Vector"])
|
||||
if texture_extension == "REPEAT":
|
||||
axes = nodes.new("ShaderNodeSeparateXYZ")
|
||||
axes.location = (-1650.0, 350.0)
|
||||
clamp = nodes.new("ShaderNodeClamp")
|
||||
clamp.location = (-1450.0, 100.0)
|
||||
half_pixel = 0.5 / max(1, int(material_image.size[1]))
|
||||
clamp.inputs["Min"].default_value = half_pixel
|
||||
clamp.inputs["Max"].default_value = 1.0 - half_pixel
|
||||
coordinates = nodes.new("ShaderNodeCombineXYZ")
|
||||
coordinates.location = (-1200.0, 350.0)
|
||||
uv_map.location = (-1900.0, 350.0)
|
||||
links.new(uv_map.outputs["UV"], axes.inputs[0])
|
||||
links.new(axes.outputs["Y"], clamp.inputs["Value"])
|
||||
links.new(axes.outputs["X"], coordinates.inputs["X"])
|
||||
links.new(clamp.outputs[0], coordinates.inputs["Y"])
|
||||
links.new(coordinates.outputs[0], color_texture.inputs["Vector"])
|
||||
alpha = color_texture.outputs["Alpha"]
|
||||
if shadeless and displacement_image is None:
|
||||
shader = nodes.new("ShaderNodeGroup")
|
||||
shader.node_tree = load_node_group(SHADELESS_NODE_GROUP_NAME)
|
||||
shader.location = (0.0, 350.0)
|
||||
shader.location = (-300.0, 300.0)
|
||||
output.location = (100.0, 300.0)
|
||||
links.new(color_texture.outputs["Color"], shader.inputs["Color"])
|
||||
links.new(alpha, shader.inputs["Alpha"])
|
||||
links.new(shader.outputs["Shader"], output.inputs["Surface"])
|
||||
@@ -148,7 +129,7 @@ def create_image_material(
|
||||
material_inputs = nodes.new("ShaderNodeGroup")
|
||||
is_depth_plane = displacement_image is not None
|
||||
material_inputs.node_tree = material_node_group(is_depth_plane)
|
||||
material_inputs.location = (-300.0, 350.0)
|
||||
material_inputs.location = (-300.0, 300.0)
|
||||
material_inputs.inputs["Bump Scale"].default_value = depth_scale
|
||||
if not is_depth_plane:
|
||||
material_inputs.inputs["Alpha Fix"].default_value = float(
|
||||
@@ -165,14 +146,13 @@ def create_image_material(
|
||||
normal_texture.image = normal_image
|
||||
normal_texture.interpolation = "Linear"
|
||||
normal_texture.extension = "EXTEND"
|
||||
normal_texture.location = (-1000.0, -50.0)
|
||||
links.new(uv_map.outputs["UV"], normal_texture.inputs["Vector"])
|
||||
normal_texture.location = (-700.0, 0.0)
|
||||
links.new(normal_texture.outputs["Color"], material_inputs.inputs["Normal"])
|
||||
if shadeless:
|
||||
shader = nodes.new("ShaderNodeGroup")
|
||||
shader.node_tree = load_node_group(SHADELESS_NODE_GROUP_NAME)
|
||||
shader.name = SHADELESS_NODE_GROUP_NAME
|
||||
shader.location = (0.0, 550.0)
|
||||
shader.location = (100.0, 300.0)
|
||||
links.new(
|
||||
material_inputs.outputs["Color"],
|
||||
shader.inputs["Color"],
|
||||
@@ -187,7 +167,7 @@ def create_image_material(
|
||||
)
|
||||
else:
|
||||
shader = nodes.new("ShaderNodeBsdfPrincipled")
|
||||
shader.location = (250.0, 400.0)
|
||||
shader.location = (100.0, 300.0)
|
||||
shader.inputs["IOR"].default_value = 1.2
|
||||
|
||||
links.new(
|
||||
@@ -223,18 +203,17 @@ def create_image_material(
|
||||
depth_texture.image = displacement_image
|
||||
depth_texture.interpolation = "Linear"
|
||||
depth_texture.extension = "EXTEND"
|
||||
depth_texture.location = (-1000.0, -400.0)
|
||||
links.new(uv_map.outputs["UV"], depth_texture.inputs["Vector"])
|
||||
depth_texture.location = (-1200.0, -300.0)
|
||||
displacement_height = depth_texture.outputs["Color"]
|
||||
if displacement_midlevel != 0.0 or displacement_scale != 1.0:
|
||||
depth_adjust = nodes.new("ShaderNodeMath")
|
||||
depth_adjust.operation = "SUBTRACT"
|
||||
depth_adjust.inputs[1].default_value = displacement_midlevel
|
||||
depth_adjust.location = (500.0, -550.0)
|
||||
depth_adjust.location = (-950.0, -300.0)
|
||||
depth_scale_node = nodes.new("ShaderNodeMath")
|
||||
depth_scale_node.operation = "MULTIPLY"
|
||||
depth_scale_node.inputs[1].default_value = displacement_scale
|
||||
depth_scale_node.location = (700.0, -550.0)
|
||||
depth_scale_node.location = (-700.0, -300.0)
|
||||
links.new(displacement_height, depth_adjust.inputs[0])
|
||||
links.new(depth_adjust.outputs[0], depth_scale_node.inputs[0])
|
||||
links.new(depth_scale_node.outputs[0], material_inputs.inputs["Depth"])
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import bpy
|
||||
|
||||
from ...common.ai import invoke_ai_setup_if_needed, production_device, report_ai_error, require_environment, require_input_path, require_model
|
||||
from ...common.image import is_image_empty, is_animated_image
|
||||
from ...common.image import is_image_empty, is_animated_image, panorama_crop_bounds
|
||||
from ...common.color_image import prepare_material_color_input, material_analysis_input, cleanup_material_color_input
|
||||
from ...preferences import addon_preferences, configured_max_ai_input_size
|
||||
from ...preferences import configured_geometry_model
|
||||
@@ -53,7 +53,7 @@ class GeneratePanorama(JobOperatorBase, bpy.types.Operator):
|
||||
class ConvertToPanorama(bpy.types.Operator):
|
||||
bl_idname = "anyimage.convert_to_panorama"
|
||||
bl_label = "Convert to Panorama"
|
||||
bl_description = "Turn a still 2:1 panoramic image into a surrounding 3D surface using AI-estimated depth."
|
||||
bl_description = "Turn a still panoramic image into a surrounding 3D surface using AI-estimated depth."
|
||||
bl_options = {"REGISTER", "UNDO"}
|
||||
|
||||
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=8, min=0, max=8)
|
||||
@@ -68,16 +68,26 @@ class ConvertToPanorama(bpy.types.Operator):
|
||||
source = context.object
|
||||
image = source.data
|
||||
width, height = image.size
|
||||
if is_animated_image(image) or height <= 0 or width != height * 2:
|
||||
self.report({"ERROR"}, "Panorama conversion requires a static 2:1 equirectangular image")
|
||||
if is_animated_image(image):
|
||||
self.report({"ERROR"}, "Panorama conversion requires a static image")
|
||||
return {"CANCELLED"}
|
||||
try:
|
||||
bounds = panorama_crop_bounds(width, height)
|
||||
except ValueError as error:
|
||||
self.report({"ERROR"}, str(error))
|
||||
return {"CANCELLED"}
|
||||
setup = invoke_ai_setup_if_needed(self)
|
||||
if setup is not None:
|
||||
return setup
|
||||
crop_width = bounds[2] - bounds[0]
|
||||
crop_height = bounds[3] - bounds[1]
|
||||
if (crop_width, crop_height) != (width, height):
|
||||
self.report({"WARNING"},
|
||||
f"Panorama input cropped from {width} x {height} to {crop_width} x {crop_height}")
|
||||
color_path = None
|
||||
try:
|
||||
preferences = addon_preferences(context)
|
||||
color_path = prepare_material_color_input(image)
|
||||
color_path = prepare_material_color_input(image, bounds)
|
||||
path = material_analysis_input(color_path)
|
||||
result = bpy.ops.anyimage.generate_panorama(
|
||||
input_path=str(path), color_path=str(color_path), source_object_name=source.name,
|
||||
|
||||
@@ -9,6 +9,7 @@ from ...common.image import (
|
||||
)
|
||||
from ...common.depth import (
|
||||
depth_uniform_scale,
|
||||
fit_image_depth_direction,
|
||||
load_depth_metadata,
|
||||
load_depth_result_image,
|
||||
)
|
||||
@@ -80,12 +81,26 @@ def create_depth_plane_object(
|
||||
modifier.node_group = node_group
|
||||
modifier.show_group_selector = False
|
||||
uniform_scale = depth_uniform_scale(source_object, depth_metadata)
|
||||
for name, value in (
|
||||
inputs = [
|
||||
("Subdivide", max(0, min(10, int(subdivisions)))),
|
||||
("Reference Depth", depth_metadata["reference_depth"] * uniform_scale),
|
||||
("Uniform Scale", uniform_scale),
|
||||
("Depth Image", depth_image),
|
||||
):
|
||||
]
|
||||
if plane_type == "RELIEF":
|
||||
try:
|
||||
depth_direction = fit_image_depth_direction(
|
||||
depth_image,
|
||||
bounds,
|
||||
uniform_scale,
|
||||
)
|
||||
except ValueError:
|
||||
depth_direction = (0.0, 0.0, 1.0)
|
||||
inputs.append((
|
||||
"Depth Direction",
|
||||
depth_direction,
|
||||
))
|
||||
for name, value in inputs:
|
||||
set_modifier_input(
|
||||
modifier,
|
||||
modifier_input_identifier(node_group, name),
|
||||
|
||||
176
src/anyimage/operators/cutout_tool/boundary_padding.py
Normal file
176
src/anyimage/operators/cutout_tool/boundary_padding.py
Normal file
@@ -0,0 +1,176 @@
|
||||
"""Extend Cutout texture boundaries along one structural inward direction."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
from scipy import ndimage
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BoundaryMapping:
|
||||
target: object
|
||||
source_y: object
|
||||
source_x: object
|
||||
|
||||
|
||||
def _resized_mask(content_values, threshold, target_shape):
|
||||
values = np.asarray(content_values, dtype=np.float32)
|
||||
if values.ndim != 2 or not values.size or not np.isfinite(values).all():
|
||||
raise ValueError("Cutout content values must be a finite two-dimensional array")
|
||||
target_height, target_width = (int(value) for value in target_shape)
|
||||
if target_height <= 0 or target_width <= 0:
|
||||
raise ValueError("Cutout padding target dimensions must be positive")
|
||||
source_height, source_width = values.shape
|
||||
y = np.minimum(
|
||||
(np.arange(target_height) * source_height / target_height).astype(np.int64),
|
||||
source_height - 1,
|
||||
)
|
||||
x = np.minimum(
|
||||
(np.arange(target_width) * source_width / target_width).astype(np.int64),
|
||||
source_width - 1,
|
||||
)
|
||||
return values[y[:, None], x[None, :]] >= float(threshold)
|
||||
|
||||
|
||||
def build_boundary_mapping(content_values, threshold, padding, target_shape=None):
|
||||
"""Map a boundary band to one fixed-distance sample along its inward normal."""
|
||||
values = np.asarray(content_values, dtype=np.float32)
|
||||
shape = values.shape if target_shape is None else tuple(target_shape)
|
||||
mask = _resized_mask(values, threshold, shape)
|
||||
height, width = mask.shape
|
||||
yy, xx = np.mgrid[:height, :width].astype(np.float32)
|
||||
source_y, source_x = yy.copy(), xx.copy()
|
||||
padding = float(padding)
|
||||
if not np.isfinite(padding) or padding < 0.0:
|
||||
raise ValueError("Cutout Boundary Padding must be finite and nonnegative")
|
||||
if padding == 0.0 or not mask.any() or mask.all():
|
||||
return BoundaryMapping(np.zeros(mask.shape, dtype=bool), source_y, source_x)
|
||||
|
||||
scale = np.sqrt(
|
||||
(height / values.shape[0]) * (width / values.shape[1])
|
||||
)
|
||||
distance = padding * scale
|
||||
signed = ndimage.distance_transform_edt(mask) - ndimage.distance_transform_edt(~mask)
|
||||
inward_y = ndimage.sobel(signed, axis=0, mode="nearest")
|
||||
inward_x = ndimage.sobel(signed, axis=1, mode="nearest")
|
||||
length = np.hypot(inward_y, inward_x)
|
||||
defined = np.isfinite(length) & (length > 1e-6)
|
||||
inward_y = np.divide(inward_y, length, out=np.zeros_like(inward_y), where=defined)
|
||||
inward_x = np.divide(inward_x, length, out=np.zeros_like(inward_x), where=defined)
|
||||
band = (signed >= -distance) & (signed <= distance) & defined
|
||||
travel = np.maximum(distance - signed, 0.0)
|
||||
|
||||
entered = mask & band
|
||||
active = band.copy()
|
||||
last_y, last_x = source_y.copy(), source_x.copy()
|
||||
step_count = max(1, int(np.ceil(float(travel[band].max(initial=0.0)) * 2.0)))
|
||||
for step in range(1, step_count + 1):
|
||||
step_distance = np.minimum(step * 0.5, travel)
|
||||
sample_y = np.clip(yy + inward_y * step_distance, 0.0, height - 1.0)
|
||||
sample_x = np.clip(xx + inward_x * step_distance, 0.0, width - 1.0)
|
||||
row = np.rint(sample_y).astype(np.int64)
|
||||
column = np.rint(sample_x).astype(np.int64)
|
||||
inside = mask[row, column]
|
||||
update = active & inside
|
||||
last_y[update] = sample_y[update]
|
||||
last_x[update] = sample_x[update]
|
||||
entered |= update
|
||||
active &= ~entered | inside
|
||||
|
||||
target = band & entered
|
||||
source_y[target] = last_y[target]
|
||||
source_x[target] = last_x[target]
|
||||
return BoundaryMapping(target, source_y, source_x)
|
||||
|
||||
|
||||
def apply_boundary_mapping(values, mapping):
|
||||
"""Sample all channels at mapped coordinates while preserving other pixels."""
|
||||
array = np.asarray(values, dtype=np.float32)
|
||||
if array.ndim != 3 or array.shape[:2] != mapping.target.shape:
|
||||
raise ValueError("Cutout texture values do not match the boundary mapping")
|
||||
result = array.copy()
|
||||
if not mapping.target.any():
|
||||
return result
|
||||
coordinates = (mapping.source_y[mapping.target], mapping.source_x[mapping.target])
|
||||
for channel in range(array.shape[2]):
|
||||
result[..., channel][mapping.target] = ndimage.map_coordinates(
|
||||
array[..., channel], coordinates, order=1, mode="nearest"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def apply_depth_boundary_mapping(rgba, mapping, intrinsics):
|
||||
"""Move camera Z and validity, then project target pixels onto their own rays."""
|
||||
array = np.asarray(rgba, dtype=np.float32)
|
||||
if array.ndim != 3 or array.shape[2] != 4 or array.shape[:2] != mapping.target.shape:
|
||||
raise ValueError("Cutout Depth values do not match the boundary mapping")
|
||||
camera = np.asarray(intrinsics, dtype=np.float32)
|
||||
if camera.shape != (3, 3) or camera[0, 0] <= 0.0 or camera[1, 1] <= 0.0:
|
||||
raise ValueError("Cutout Depth padding requires valid camera intrinsics")
|
||||
result = array.copy()
|
||||
if not mapping.target.any():
|
||||
return result
|
||||
coordinates = (mapping.source_y[mapping.target], mapping.source_x[mapping.target])
|
||||
depth = ndimage.map_coordinates(array[..., 2], coordinates, order=1, mode="nearest")
|
||||
validity = ndimage.map_coordinates(array[..., 3], coordinates, order=1, mode="nearest")
|
||||
target_y, target_x = np.nonzero(mapping.target)
|
||||
usable = np.isfinite(depth) & (depth > 0.0) & np.isfinite(validity)
|
||||
target_y, target_x = target_y[usable], target_x[usable]
|
||||
depth, validity = depth[usable], validity[usable]
|
||||
result[target_y, target_x, 0] = (
|
||||
(target_x - camera[0, 2]) / camera[0, 0] * depth
|
||||
)
|
||||
result[target_y, target_x, 1] = (
|
||||
(target_y - camera[1, 2]) / camera[1, 1] * depth
|
||||
)
|
||||
result[target_y, target_x, 2] = depth
|
||||
result[target_y, target_x, 3] = validity
|
||||
return result
|
||||
|
||||
|
||||
def image_rgba_buffer(image):
|
||||
"""Read one Blender Image buffer as a top-down RGBA array without reinterpretation."""
|
||||
width, height = (int(value) for value in image.size)
|
||||
pixels = np.empty(width * height * 4, dtype=np.float32)
|
||||
image.pixels.foreach_get(pixels)
|
||||
return np.flipud(pixels.reshape((height, width, 4)))
|
||||
|
||||
|
||||
def write_image_rgba_buffer(image, rgba):
|
||||
"""Replace and pack one Blender Image buffer."""
|
||||
values = np.asarray(rgba, dtype=np.float32)
|
||||
width, height = (int(value) for value in image.size)
|
||||
if values.shape != (height, width, 4) or not np.isfinite(values).all():
|
||||
raise ValueError("Cutout texture padding produced invalid RGBA")
|
||||
image.pixels.foreach_set(np.flipud(values).ravel())
|
||||
image.update()
|
||||
image.pack()
|
||||
|
||||
|
||||
def pad_cutout_images(
|
||||
color_image,
|
||||
depth_image,
|
||||
depth_metadata,
|
||||
content_values,
|
||||
alpha_threshold,
|
||||
padding,
|
||||
):
|
||||
"""Apply one structural Cutout boundary rule before image consumption."""
|
||||
padding = float(padding)
|
||||
if padding <= 0.0:
|
||||
return
|
||||
color = image_rgba_buffer(color_image)
|
||||
color_mapping = build_boundary_mapping(
|
||||
content_values, alpha_threshold, padding, color.shape[:2]
|
||||
)
|
||||
write_image_rgba_buffer(color_image, apply_boundary_mapping(color, color_mapping))
|
||||
if depth_image is None:
|
||||
return
|
||||
depth = image_rgba_buffer(depth_image)
|
||||
depth_mapping = build_boundary_mapping(
|
||||
content_values, alpha_threshold, padding, depth.shape[:2]
|
||||
)
|
||||
write_image_rgba_buffer(
|
||||
depth_image,
|
||||
apply_depth_boundary_mapping(depth, depth_mapping, depth_metadata["intrinsics"]),
|
||||
)
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Calibrate metric reference depth and the Depth Symmetry base plane."""
|
||||
|
||||
PLANE_VARIANCE_FLOOR_RATIO = 0.025
|
||||
from ...common.depth import fit_depth_direction
|
||||
|
||||
|
||||
DEPTH_REFERENCE_PERCENTILE = 95.0
|
||||
ADAPTIVE_DEPTH_TRANSPARENCY_RATIO = 0.05
|
||||
DEFAULT_DEPTH_BASELINE = 8.0
|
||||
@@ -79,49 +81,6 @@ def _sample_valid_bilinear(values, valid, x, y):
|
||||
return result, usable
|
||||
|
||||
|
||||
def _weighted_plane(coordinates, values, weights):
|
||||
import numpy as np
|
||||
|
||||
total = float(np.sum(weights))
|
||||
if total <= 1e-12:
|
||||
return np.zeros(3, dtype=np.float32)
|
||||
center = np.sum(coordinates * weights[:, None], axis=0) / total
|
||||
value_center = float(np.sum(values * weights) / total)
|
||||
centered = coordinates - center
|
||||
covariance = (centered * weights[:, None]).T @ centered / total
|
||||
cross = centered.T @ (weights * (values - value_center)) / total
|
||||
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
|
||||
largest = max(float(eigenvalues[-1]), 0.0)
|
||||
if largest <= 1e-12:
|
||||
slopes = np.zeros(2, dtype=np.float64)
|
||||
else:
|
||||
supported = np.maximum(eigenvalues, largest * PLANE_VARIANCE_FLOOR_RATIO)
|
||||
slopes = eigenvectors @ ((eigenvectors.T @ cross) / supported)
|
||||
intercept = value_center - float(center @ slopes)
|
||||
return np.asarray((*slopes, intercept), dtype=np.float32)
|
||||
|
||||
|
||||
def _stable_plane(coordinates, values):
|
||||
import numpy as np
|
||||
|
||||
coordinates = np.asarray(coordinates, dtype=np.float64)
|
||||
values = np.asarray(values, dtype=np.float64)
|
||||
if len(coordinates) < 2:
|
||||
return np.asarray((0.0, 0.0, np.median(values)), dtype=np.float32)
|
||||
weights = np.ones(len(values), dtype=np.float64)
|
||||
plane = np.zeros(3, dtype=np.float32)
|
||||
for _iteration in range(4):
|
||||
plane = _weighted_plane(coordinates, values, weights)
|
||||
residual = values - coordinates @ plane[:2] - plane[2]
|
||||
centered = residual - np.median(residual)
|
||||
scale = max(float(np.median(np.abs(centered))) * 1.4826, 1e-6)
|
||||
weights = np.minimum(
|
||||
1.0,
|
||||
(2.5 * scale) / np.maximum(np.abs(centered), 1e-12),
|
||||
)
|
||||
return plane
|
||||
|
||||
|
||||
def depth_symmetry_projection_parameters(
|
||||
vertices,
|
||||
vertex_uv,
|
||||
@@ -157,13 +116,15 @@ def depth_symmetry_projection_parameters(
|
||||
if not usable.any():
|
||||
raise ValueError("MoGe-2 found no valid Depth Symmetry samples")
|
||||
local_coordinates = vertices[:, :2]
|
||||
metric_depth = sampled[usable] * float(uniform_scale)
|
||||
plane = _stable_plane(local_coordinates[usable], metric_depth)
|
||||
reference = metric_depth_reference(
|
||||
camera_depth,
|
||||
calibration_valid,
|
||||
uniform_scale,
|
||||
)
|
||||
direction = np.asarray((plane[0], plane[1], 1.0), dtype=np.float64)
|
||||
direction /= np.linalg.norm(direction)
|
||||
return reference, tuple(float(value) for value in direction)
|
||||
direction = fit_depth_direction(
|
||||
local_coordinates,
|
||||
sampled,
|
||||
usable,
|
||||
uniform_scale,
|
||||
)
|
||||
return reference, direction
|
||||
|
||||
@@ -473,6 +473,7 @@ class BaseShape:
|
||||
faces: object
|
||||
uv: object
|
||||
balloon: object
|
||||
mask: object
|
||||
|
||||
|
||||
def cutout_domain(mesh, bounds, world_metric, balloon_algorithm="POISSON"):
|
||||
@@ -664,14 +665,16 @@ def build_base_shape(
|
||||
) @ metric_transform(world_metric)
|
||||
from .mesh import build_mesh
|
||||
|
||||
mesh_result = build_mesh(
|
||||
content_values,
|
||||
edge_length,
|
||||
alpha_threshold,
|
||||
pixel_transform=pixel_transform,
|
||||
fine_outline=fine_outline,
|
||||
)
|
||||
*mesh_geometry, mask = mesh_result
|
||||
mesh = (
|
||||
*build_mesh(
|
||||
content_values,
|
||||
edge_length,
|
||||
alpha_threshold,
|
||||
pixel_transform=pixel_transform,
|
||||
fine_outline=fine_outline,
|
||||
),
|
||||
*mesh_geometry,
|
||||
(content_values.shape[1], content_values.shape[0]),
|
||||
)
|
||||
preferences = addon_preferences(context)
|
||||
@@ -683,4 +686,5 @@ def build_base_shape(
|
||||
faces=counter_clockwise_faces(planar, faces),
|
||||
uv=np.asarray(uv, dtype=np.float32),
|
||||
balloon=np.asarray(balloon, dtype=np.float64) / world_z_scale,
|
||||
mask=np.asarray(mask, dtype=bool),
|
||||
)
|
||||
|
||||
@@ -154,8 +154,15 @@ def prepare_mask(alpha, threshold, transform, fine_outline):
|
||||
return mask
|
||||
|
||||
|
||||
def build_mesh(alpha, spacing, threshold, *, pixel_transform=None, fine_outline=True):
|
||||
"""Return pixel-space quality triangles and component ranges."""
|
||||
def build_mesh(
|
||||
alpha,
|
||||
spacing,
|
||||
threshold,
|
||||
*,
|
||||
pixel_transform=None,
|
||||
fine_outline=True,
|
||||
):
|
||||
"""Return pixel-space quality triangles, component ranges and final Mask."""
|
||||
alpha = np.asarray(alpha, dtype=float)
|
||||
if alpha.ndim != 2 or not alpha.size or not np.isfinite(alpha).all():
|
||||
raise ValueError("The Cutout Alpha must be a finite two-dimensional array")
|
||||
@@ -210,7 +217,8 @@ def build_mesh(alpha, spacing, threshold, *, pixel_transform=None, fine_outline=
|
||||
if offset > 50000:
|
||||
raise ValueError("Cutout mesh vertex limit exceeded")
|
||||
points, faces = np.concatenate(points), np.concatenate(faces)
|
||||
return points, faces, g.boundary_vertices_from_faces(len(points), faces), components
|
||||
result = points, faces, g.boundary_vertices_from_faces(len(points), faces), components
|
||||
return *result, mask
|
||||
|
||||
|
||||
def filter_alpha(alpha, threshold, min_area=16):
|
||||
|
||||
@@ -238,7 +238,7 @@ def create_shape_object(
|
||||
symmetry.show_group_selector = False
|
||||
set_modifier_input(
|
||||
symmetry,
|
||||
modifier_input_identifier(symmetry_group, "Symmetry Direction"),
|
||||
modifier_input_identifier(symmetry_group, "Direction"),
|
||||
depth_direction,
|
||||
)
|
||||
return finalize_object_result(
|
||||
|
||||
@@ -5,7 +5,11 @@ import json
|
||||
import bpy
|
||||
|
||||
from ...preferences import (
|
||||
DEFAULT_CUTOUT_BOUNDARY_PADDING,
|
||||
MAX_CUTOUT_BOUNDARY_PADDING,
|
||||
MIN_CUTOUT_BOUNDARY_PADDING,
|
||||
addon_preferences,
|
||||
configured_cutout_boundary_padding,
|
||||
configured_geometry_model,
|
||||
configured_max_ai_input_size,
|
||||
)
|
||||
@@ -67,6 +71,7 @@ from .object import (
|
||||
create_shape_object,
|
||||
)
|
||||
from .shape import CUTOUT_SHAPES, DEPTH_CUTOUT_SHAPES
|
||||
from .boundary_padding import pad_cutout_images
|
||||
|
||||
_HIDDEN = {"HIDDEN", "SKIP_SAVE"}
|
||||
|
||||
@@ -97,6 +102,7 @@ def create_cutout_shape(
|
||||
depth_metadata=None,
|
||||
normal_image=None,
|
||||
normal_mode=NormalMode.NONE,
|
||||
boundary_padding=DEFAULT_CUTOUT_BOUNDARY_PADDING,
|
||||
color_image,
|
||||
gesture="LASSO",
|
||||
report=None,
|
||||
@@ -120,6 +126,14 @@ def create_cutout_shape(
|
||||
alpha_threshold=alpha_threshold,
|
||||
fine_outline=fine_outline,
|
||||
)
|
||||
pad_cutout_images(
|
||||
color_image,
|
||||
depth_image,
|
||||
depth_metadata,
|
||||
base_shape.mask,
|
||||
0.5,
|
||||
boundary_padding,
|
||||
)
|
||||
result = create_shape_object(
|
||||
context,
|
||||
source_object,
|
||||
@@ -220,6 +234,12 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
|
||||
generate_normal: bpy.props.BoolProperty(default=False, options=_HIDDEN)
|
||||
alpha_threshold: cutout_alpha_threshold_property()
|
||||
fine_outline: cutout_fine_outline_property()
|
||||
boundary_padding: bpy.props.FloatProperty(
|
||||
default=DEFAULT_CUTOUT_BOUNDARY_PADDING,
|
||||
min=MIN_CUTOUT_BOUNDARY_PADDING,
|
||||
max=MAX_CUTOUT_BOUNDARY_PADDING,
|
||||
options=_HIDDEN,
|
||||
)
|
||||
def _needs_generation(self):
|
||||
return self.shape in DEPTH_CUTOUT_SHAPES or self.generate_normal
|
||||
|
||||
@@ -240,6 +260,7 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
|
||||
context.view_layer.objects.active = source_object
|
||||
source_object.select_set(True)
|
||||
self.max_ai_input_size = configured_max_ai_input_size(context)
|
||||
self.boundary_padding = configured_cutout_boundary_padding(context)
|
||||
if not self._needs_generation():
|
||||
left, top, right, bottom = selection_mask.bounds
|
||||
content_values = _cutout_content_values(
|
||||
@@ -253,6 +274,7 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
|
||||
content_values, selection_mask.bounds,
|
||||
alpha_threshold=self.alpha_threshold,
|
||||
fine_outline=self.fine_outline, color_image=color_image,
|
||||
boundary_padding=self.boundary_padding,
|
||||
gesture=self.gesture,
|
||||
report=self.report,
|
||||
)
|
||||
@@ -362,6 +384,7 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
|
||||
normal_mode=normal_mode,
|
||||
alpha_threshold=self.alpha_threshold,
|
||||
fine_outline=self.fine_outline,
|
||||
boundary_padding=self.boundary_padding,
|
||||
color_image=color_image,
|
||||
gesture=self.gesture,
|
||||
report=self.report,
|
||||
|
||||
@@ -19,6 +19,9 @@ from .server.model_catalog import (
|
||||
DEFAULT_MAX_FRAME_RESOLUTION = 2048
|
||||
MIN_FRAME_RESOLUTION = 64
|
||||
MAX_FRAME_RESOLUTION = 8192
|
||||
DEFAULT_CUTOUT_BOUNDARY_PADDING = 2.0
|
||||
MIN_CUTOUT_BOUNDARY_PADDING = 1.0
|
||||
MAX_CUTOUT_BOUNDARY_PADDING = 4.0
|
||||
|
||||
ADDON_PACKAGE = __package__
|
||||
DEFAULT_STORAGE_ROOT = Path.home() / ".anyimage"
|
||||
@@ -89,6 +92,17 @@ def configured_min_cutout_relative_edge_length(context=None):
|
||||
return min(max(percent, 0.001), 100.0) / 100.0
|
||||
|
||||
|
||||
def configured_cutout_boundary_padding(context=None):
|
||||
value = float(
|
||||
getattr(
|
||||
addon_preferences(context),
|
||||
"cutout_boundary_padding",
|
||||
DEFAULT_CUTOUT_BOUNDARY_PADDING,
|
||||
)
|
||||
)
|
||||
return min(max(value, MIN_CUTOUT_BOUNDARY_PADDING), MAX_CUTOUT_BOUNDARY_PADDING)
|
||||
|
||||
|
||||
def configured_max_frame_resolution(context=None):
|
||||
preferences = addon_preferences(context)
|
||||
value = int(
|
||||
@@ -174,6 +188,14 @@ class AnyImagePreferences(bpy.types.AddonPreferences):
|
||||
soft_max=10.0,
|
||||
subtype="PERCENTAGE",
|
||||
)
|
||||
cutout_boundary_padding: bpy.props.FloatProperty(
|
||||
name="Boundary Padding",
|
||||
description="Sample inward and extend Cutout color and depth at the outline",
|
||||
default=DEFAULT_CUTOUT_BOUNDARY_PADDING,
|
||||
min=MIN_CUTOUT_BOUNDARY_PADDING,
|
||||
max=MAX_CUTOUT_BOUNDARY_PADDING,
|
||||
subtype="PIXEL",
|
||||
)
|
||||
max_frame_resolution: bpy.props.IntProperty(
|
||||
name="Maximum Frame Resolution",
|
||||
description="Set the maximum length, in pixels, of the combined image's longest edge.",
|
||||
@@ -278,6 +300,7 @@ class AnyImagePreferences(bpy.types.AddonPreferences):
|
||||
tools.separator(type="LINE")
|
||||
tools.label(text="Cutout Tool")
|
||||
tools.prop(self, "min_cutout_relative_edge_length")
|
||||
tools.prop(self, "cutout_boundary_padding")
|
||||
tools.prop(self, "balloon_profile_algorithm")
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
"""Extend reliable depth along each visible component's silhouette."""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
|
||||
def extend_edge_depth(frame, alpha, alpha_threshold=0.9):
|
||||
"""Return edge-corrected geometry while preserving reliable camera points."""
|
||||
import numpy as np
|
||||
from scipy import ndimage
|
||||
|
||||
from ..models.onnx_moge2 import depth_to_points
|
||||
|
||||
depth = frame.depth
|
||||
alpha = np.asarray(alpha, dtype=np.float32)
|
||||
if alpha.shape != depth.shape:
|
||||
raise ValueError("Edge Alpha must match the depth dimensions")
|
||||
if np.all(alpha == 1.0):
|
||||
return frame
|
||||
|
||||
visible = (alpha > 0.0) & (alpha >= alpha_threshold)
|
||||
opaque = visible & (alpha >= 0.95)
|
||||
candidates = (
|
||||
opaque
|
||||
& (frame.validity > 0.5)
|
||||
& np.isfinite(depth)
|
||||
& (depth > 0.0)
|
||||
)
|
||||
if not candidates.any():
|
||||
return frame
|
||||
interior_distance = ndimage.distance_transform_edt(
|
||||
np.pad(opaque, 3, mode="edge")
|
||||
)[3:-3, 3:-3]
|
||||
interior = opaque & (interior_distance > 2.0)
|
||||
core = candidates & interior
|
||||
del interior_distance
|
||||
labels, _ = ndimage.label(visible, structure=np.ones((3, 3), dtype=bool))
|
||||
corrected = depth.copy()
|
||||
background_distance = np.full(depth.shape, np.inf, dtype=np.float32)
|
||||
|
||||
for component_id, bounds in enumerate(ndimage.find_objects(labels), start=1):
|
||||
region = tuple(
|
||||
slice(max(0, bound.start - 7), min(size, bound.stop + 7))
|
||||
for bound, size in zip(bounds, depth.shape)
|
||||
)
|
||||
component = labels[region] == component_id
|
||||
donors = core[region] & component
|
||||
if not donors.any():
|
||||
donors = candidates[region] & component
|
||||
if not donors.any():
|
||||
continue
|
||||
|
||||
indices = ndimage.distance_transform_edt(
|
||||
~donors, return_distances=False, return_indices=True
|
||||
)
|
||||
extended = depth[region][tuple(indices)]
|
||||
smoothed = ndimage.gaussian_filter(extended, sigma=0.8, radius=3)
|
||||
distance = ndimage.distance_transform_edt(~component)
|
||||
background = (
|
||||
~visible[region]
|
||||
& (distance <= 4.0)
|
||||
& (distance < background_distance[region])
|
||||
)
|
||||
target = (component & ~interior[region] & ~donors) | background
|
||||
corrected[region][target] = smoothed[target]
|
||||
background_distance[region][background] = distance[background]
|
||||
|
||||
changed = corrected != depth
|
||||
if not changed.any():
|
||||
return frame
|
||||
points = frame.points
|
||||
if points is not None:
|
||||
height, width = depth.shape
|
||||
intrinsics = frame.intrinsics.copy()
|
||||
intrinsics[0] /= width
|
||||
intrinsics[1] /= height
|
||||
projected = depth_to_points(corrected, intrinsics)
|
||||
points = points.copy()
|
||||
points[changed] = projected[changed]
|
||||
return replace(frame, depth=corrected, points=points)
|
||||
@@ -8,7 +8,6 @@ from .depth_texture import (
|
||||
write_depth_texture,
|
||||
)
|
||||
from .normal_texture import write_camera_normal_texture
|
||||
from .edge_depth import extend_edge_depth
|
||||
|
||||
|
||||
def _limited_model_input(source_path, output_path, max_input_size):
|
||||
@@ -54,7 +53,6 @@ def generate_moge_artifacts(
|
||||
generate_depth=False,
|
||||
normal_mode="NONE",
|
||||
alpha_threshold=0.9,
|
||||
extend_depth_edges=False,
|
||||
):
|
||||
"""Run MoGe once and write only the requested file artifacts."""
|
||||
normal_mode = str(normal_mode).upper()
|
||||
@@ -77,15 +75,11 @@ def generate_moge_artifacts(
|
||||
if generate_depth:
|
||||
context.progress(0.45, "Generating depth texture")
|
||||
alpha = _image_alpha(input_path, frame.depth.shape)
|
||||
depth_frame = (
|
||||
extend_edge_depth(frame, alpha, alpha_threshold)
|
||||
if extend_depth_edges else frame
|
||||
)
|
||||
result["depth"] = write_depth_texture(
|
||||
depth_frame, context.directory / "depth.exr", alpha=alpha,
|
||||
frame, context.directory / "depth.exr", alpha=alpha,
|
||||
)
|
||||
result["depth_metadata"] = write_depth_metadata(
|
||||
depth_frame,
|
||||
frame,
|
||||
(alpha > 0.0) & (alpha >= alpha_threshold),
|
||||
context.directory / "depth.json",
|
||||
)
|
||||
|
||||
@@ -18,7 +18,6 @@ def run(context, parameters):
|
||||
parameters,
|
||||
input_path,
|
||||
generate_depth=generate_depth,
|
||||
extend_depth_edges=generate_depth,
|
||||
normal_mode=normal_mode,
|
||||
alpha_threshold=float(parameters.get("alpha_threshold", 0.9)),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user