Add cutout profile and teddy balloon height calculations
- Implemented cotangent Poisson field and Poisson balloon profile in cutout_profile.py. - Added teddy balloon heights calculation and related functions in cutout_teddy.py. - Introduced interaction utilities for image editing in interaction.py. - Developed depth normalization and analysis functions in cutout_depth.py. - Created a validation script for cutout assets in validate_cutout_asset.py.
This commit is contained in:
226
src/anyimage/operators/cutout_depth.py
Normal file
226
src/anyimage/operators/cutout_depth.py
Normal file
@@ -0,0 +1,226 @@
|
||||
"""MoGe-2 depth sampling and Depth Balloon base-plane controls."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import bpy
|
||||
|
||||
|
||||
PLANE_VARIANCE_FLOOR_RATIO = 0.025
|
||||
|
||||
|
||||
def load_depth_balloon_data(path):
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
with np.load(Path(path), allow_pickle=False) as archive:
|
||||
required = {
|
||||
"depth_signal",
|
||||
"baseline_vector",
|
||||
"valid",
|
||||
"origin",
|
||||
"reference_depth",
|
||||
"crop_size",
|
||||
"source_size",
|
||||
}
|
||||
missing = required.difference(archive.files)
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Depth Balloon data is missing: {', '.join(sorted(missing))}"
|
||||
)
|
||||
data = {
|
||||
"depth_signal": np.asarray(
|
||||
archive["depth_signal"],
|
||||
dtype=np.float32,
|
||||
),
|
||||
"baseline_vector": np.asarray(
|
||||
archive["baseline_vector"],
|
||||
dtype=np.float32,
|
||||
),
|
||||
"valid": np.asarray(archive["valid"], dtype=bool),
|
||||
"origin": tuple(int(value) for value in archive["origin"]),
|
||||
"reference_depth": float(archive["reference_depth"]),
|
||||
"crop_size": tuple(int(value) for value in archive["crop_size"]),
|
||||
"source_size": tuple(int(value) for value in archive["source_size"]),
|
||||
}
|
||||
except OSError as error:
|
||||
raise ValueError(f"Unable to read Depth Balloon data: {path}") from error
|
||||
if data["depth_signal"].ndim != 2:
|
||||
raise ValueError("Depth Balloon signal must be two-dimensional")
|
||||
if data["valid"].shape != data["depth_signal"].shape:
|
||||
raise ValueError("Depth Balloon mask dimensions do not match")
|
||||
if data["baseline_vector"].shape != (3,):
|
||||
raise ValueError("Depth Balloon base plane vector must have three components")
|
||||
if any(value <= 0 for value in (*data["crop_size"], *data["source_size"])):
|
||||
raise ValueError("Depth Balloon image dimensions must be positive")
|
||||
return data
|
||||
|
||||
|
||||
def _sample_valid_bilinear(values, valid, x, y):
|
||||
import numpy as np
|
||||
|
||||
height, width = values.shape
|
||||
x = np.clip(x, 0.0, max(width - 1, 0))
|
||||
y = np.clip(y, 0.0, max(height - 1, 0))
|
||||
x0 = np.floor(x).astype(np.int64)
|
||||
y0 = np.floor(y).astype(np.int64)
|
||||
x1 = np.minimum(x0 + 1, width - 1)
|
||||
y1 = np.minimum(y0 + 1, height - 1)
|
||||
fx = x - x0
|
||||
fy = y - y0
|
||||
samples = np.stack(
|
||||
(
|
||||
values[y0, x0],
|
||||
values[y0, x1],
|
||||
values[y1, x0],
|
||||
values[y1, x1],
|
||||
),
|
||||
axis=1,
|
||||
)
|
||||
weights = np.stack(
|
||||
(
|
||||
(1.0 - fx) * (1.0 - fy),
|
||||
fx * (1.0 - fy),
|
||||
(1.0 - fx) * fy,
|
||||
fx * fy,
|
||||
),
|
||||
axis=1,
|
||||
)
|
||||
sample_valid = np.stack(
|
||||
(
|
||||
valid[y0, x0],
|
||||
valid[y0, x1],
|
||||
valid[y1, x0],
|
||||
valid[y1, x1],
|
||||
),
|
||||
axis=1,
|
||||
)
|
||||
weights *= sample_valid
|
||||
totals = weights.sum(axis=1)
|
||||
result = np.zeros(len(x), dtype=np.float32)
|
||||
usable = totals > 1e-8
|
||||
result[usable] = (samples[usable] * weights[usable]).sum(axis=1) / totals[usable]
|
||||
return result, usable
|
||||
|
||||
|
||||
def _stable_plane(coordinates, values):
|
||||
import numpy as np
|
||||
|
||||
coordinates = np.asarray(coordinates, dtype=np.float64)
|
||||
values = np.asarray(values, dtype=np.float64)
|
||||
if len(coordinates) < 2:
|
||||
return np.zeros(3, dtype=np.float32)
|
||||
center = coordinates.mean(axis=0)
|
||||
value_center = float(values.mean())
|
||||
centered = coordinates - center
|
||||
covariance = centered.T @ centered / len(centered)
|
||||
cross = centered.T @ (values - value_center) / len(centered)
|
||||
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
|
||||
largest = max(float(eigenvalues[-1]), 0.0)
|
||||
if largest <= 1e-12:
|
||||
slopes = np.zeros(2, dtype=np.float64)
|
||||
else:
|
||||
supported = np.maximum(eigenvalues, largest * PLANE_VARIANCE_FLOOR_RATIO)
|
||||
slopes = eigenvectors @ ((eigenvectors.T @ cross) / supported)
|
||||
intercept = value_center - float(center @ slopes)
|
||||
return np.asarray((*slopes, intercept), dtype=np.float32)
|
||||
|
||||
|
||||
def depth_balloon_fields(vertices, vertex_uv, depth_data, depth_scale):
|
||||
"""Sample camera depth and fit its base plane at a fixed local scale."""
|
||||
import numpy as np
|
||||
|
||||
vertices = np.asarray(vertices, dtype=np.float32)
|
||||
uv = np.asarray(vertex_uv, dtype=np.float32)
|
||||
if vertices.ndim != 2 or vertices.shape[1] != 3:
|
||||
raise ValueError("Depth Balloon vertices must have three components")
|
||||
if uv.shape != (len(vertices), 2):
|
||||
raise ValueError("Depth Balloon UV coordinates do not match its vertices")
|
||||
depth_signal = np.asarray(depth_data["depth_signal"], dtype=np.float32)
|
||||
valid = np.asarray(depth_data["valid"], dtype=bool)
|
||||
crop_width, crop_height = depth_data["crop_size"]
|
||||
depth_height, depth_width = depth_signal.shape
|
||||
|
||||
crop_x = uv[:, 0] * max(crop_width - 1, 0)
|
||||
crop_y = (1.0 - uv[:, 1]) * max(crop_height - 1, 0)
|
||||
sample_x = crop_x * max(depth_width - 1, 0) / max(crop_width - 1, 1)
|
||||
sample_y = crop_y * max(depth_height - 1, 0) / max(crop_height - 1, 1)
|
||||
sampled, usable = _sample_valid_bilinear(
|
||||
depth_signal,
|
||||
valid,
|
||||
sample_x,
|
||||
sample_y,
|
||||
)
|
||||
inside_crop = (
|
||||
(crop_x >= 0.0)
|
||||
& (crop_x <= max(crop_width - 1, 0))
|
||||
& (crop_y >= 0.0)
|
||||
& (crop_y <= max(crop_height - 1, 0))
|
||||
)
|
||||
usable &= inside_crop
|
||||
sampled[~usable] = 0.0
|
||||
coordinates = np.stack(
|
||||
(
|
||||
sample_x / max(depth_width - 1, 1) * 2.0 - 1.0,
|
||||
sample_y / max(depth_height - 1, 1) * 2.0 - 1.0,
|
||||
np.ones(len(vertices), dtype=np.float32),
|
||||
),
|
||||
axis=1,
|
||||
).astype(np.float32)
|
||||
|
||||
depth_scale = float(depth_scale)
|
||||
if not depth_scale > 0.0:
|
||||
raise ValueError("Depth Balloon reference scale must be positive")
|
||||
normalized_plane = np.asarray(
|
||||
depth_data["baseline_vector"],
|
||||
dtype=np.float32,
|
||||
)
|
||||
baseline_values = (
|
||||
coordinates @ normalized_plane * depth_scale * 0.5
|
||||
)
|
||||
local_coordinates = vertices[:, :2]
|
||||
if np.count_nonzero(usable) >= 3:
|
||||
local_plane = _stable_plane(
|
||||
local_coordinates[usable],
|
||||
baseline_values[usable],
|
||||
)
|
||||
else:
|
||||
local_plane = np.zeros(3, dtype=np.float32)
|
||||
depth_values = sampled * depth_scale
|
||||
if usable.any():
|
||||
plane_without_offset = local_coordinates @ local_plane[:2]
|
||||
residual = depth_values - plane_without_offset
|
||||
depth_values[usable] -= float(np.median(residual[usable]))
|
||||
return depth_values, local_plane
|
||||
|
||||
|
||||
def orient_depth_balloon_plane(local_plane):
|
||||
slope_x, slope_y, offset = (float(value) for value in local_plane)
|
||||
return -slope_x, slope_y, offset
|
||||
|
||||
|
||||
def create_depth_balloon_base_plane(context, mesh_object, vertices, local_plane):
|
||||
import numpy as np
|
||||
from mathutils import Vector
|
||||
|
||||
vertices = np.asarray(vertices, dtype=np.float32)
|
||||
slope_y, slope_z, offset = (float(value) for value in local_plane)
|
||||
extent = vertices[:, 1:].max(axis=0) - vertices[:, 1:].min(axis=0)
|
||||
center_y = 0.0
|
||||
center_z = 0.0
|
||||
center_x = offset
|
||||
normal = Vector((-1.0, slope_y, slope_z)).normalized()
|
||||
|
||||
controller = bpy.data.objects.new(f"{mesh_object.name} Base Plane", None)
|
||||
context.collection.objects.link(controller)
|
||||
controller.parent = mesh_object
|
||||
controller.location = (float(center_x), float(center_y), float(center_z))
|
||||
controller.rotation_euler = normal.to_track_quat("Z", "Y").to_euler()
|
||||
controller.empty_display_type = "SINGLE_ARROW"
|
||||
controller.empty_display_size = max(float(extent.max()) * 0.15, 0.01)
|
||||
controller.show_in_front = True
|
||||
controller.hide_render = True
|
||||
controller["anyimage_source_type"] = "DEPTH_BALLOON_BASE_PLANE"
|
||||
controller["anyimage_base_plane"] = (slope_y, slope_z, offset)
|
||||
return controller
|
||||
|
||||
|
||||
Reference in New Issue
Block a user