nan
7d39fe8c5e
- Implemented cotangent Poisson field and Poisson balloon profile in cutout_profile.py. - Added teddy balloon heights calculation and related functions in cutout_teddy.py. - Introduced interaction utilities for image editing in interaction.py. - Developed depth normalization and analysis functions in cutout_depth.py. - Created a validation script for cutout assets in validate_cutout_asset.py.
227 lines
7.8 KiB
Python
227 lines
7.8 KiB
Python
"""MoGe-2 depth sampling and Depth Balloon base-plane controls."""
|
|
|
|
from pathlib import Path
|
|
|
|
import bpy
|
|
|
|
|
|
PLANE_VARIANCE_FLOOR_RATIO = 0.025
|
|
|
|
|
|
def load_depth_balloon_data(path):
|
|
import numpy as np
|
|
|
|
try:
|
|
with np.load(Path(path), allow_pickle=False) as archive:
|
|
required = {
|
|
"depth_signal",
|
|
"baseline_vector",
|
|
"valid",
|
|
"origin",
|
|
"reference_depth",
|
|
"crop_size",
|
|
"source_size",
|
|
}
|
|
missing = required.difference(archive.files)
|
|
if missing:
|
|
raise ValueError(
|
|
f"Depth Balloon data is missing: {', '.join(sorted(missing))}"
|
|
)
|
|
data = {
|
|
"depth_signal": np.asarray(
|
|
archive["depth_signal"],
|
|
dtype=np.float32,
|
|
),
|
|
"baseline_vector": np.asarray(
|
|
archive["baseline_vector"],
|
|
dtype=np.float32,
|
|
),
|
|
"valid": np.asarray(archive["valid"], dtype=bool),
|
|
"origin": tuple(int(value) for value in archive["origin"]),
|
|
"reference_depth": float(archive["reference_depth"]),
|
|
"crop_size": tuple(int(value) for value in archive["crop_size"]),
|
|
"source_size": tuple(int(value) for value in archive["source_size"]),
|
|
}
|
|
except OSError as error:
|
|
raise ValueError(f"Unable to read Depth Balloon data: {path}") from error
|
|
if data["depth_signal"].ndim != 2:
|
|
raise ValueError("Depth Balloon signal must be two-dimensional")
|
|
if data["valid"].shape != data["depth_signal"].shape:
|
|
raise ValueError("Depth Balloon mask dimensions do not match")
|
|
if data["baseline_vector"].shape != (3,):
|
|
raise ValueError("Depth Balloon base plane vector must have three components")
|
|
if any(value <= 0 for value in (*data["crop_size"], *data["source_size"])):
|
|
raise ValueError("Depth Balloon image dimensions must be positive")
|
|
return data
|
|
|
|
|
|
def _sample_valid_bilinear(values, valid, x, y):
|
|
import numpy as np
|
|
|
|
height, width = values.shape
|
|
x = np.clip(x, 0.0, max(width - 1, 0))
|
|
y = np.clip(y, 0.0, max(height - 1, 0))
|
|
x0 = np.floor(x).astype(np.int64)
|
|
y0 = np.floor(y).astype(np.int64)
|
|
x1 = np.minimum(x0 + 1, width - 1)
|
|
y1 = np.minimum(y0 + 1, height - 1)
|
|
fx = x - x0
|
|
fy = y - y0
|
|
samples = np.stack(
|
|
(
|
|
values[y0, x0],
|
|
values[y0, x1],
|
|
values[y1, x0],
|
|
values[y1, x1],
|
|
),
|
|
axis=1,
|
|
)
|
|
weights = np.stack(
|
|
(
|
|
(1.0 - fx) * (1.0 - fy),
|
|
fx * (1.0 - fy),
|
|
(1.0 - fx) * fy,
|
|
fx * fy,
|
|
),
|
|
axis=1,
|
|
)
|
|
sample_valid = np.stack(
|
|
(
|
|
valid[y0, x0],
|
|
valid[y0, x1],
|
|
valid[y1, x0],
|
|
valid[y1, x1],
|
|
),
|
|
axis=1,
|
|
)
|
|
weights *= sample_valid
|
|
totals = weights.sum(axis=1)
|
|
result = np.zeros(len(x), dtype=np.float32)
|
|
usable = totals > 1e-8
|
|
result[usable] = (samples[usable] * weights[usable]).sum(axis=1) / totals[usable]
|
|
return result, usable
|
|
|
|
|
|
def _stable_plane(coordinates, values):
|
|
import numpy as np
|
|
|
|
coordinates = np.asarray(coordinates, dtype=np.float64)
|
|
values = np.asarray(values, dtype=np.float64)
|
|
if len(coordinates) < 2:
|
|
return np.zeros(3, dtype=np.float32)
|
|
center = coordinates.mean(axis=0)
|
|
value_center = float(values.mean())
|
|
centered = coordinates - center
|
|
covariance = centered.T @ centered / len(centered)
|
|
cross = centered.T @ (values - value_center) / len(centered)
|
|
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
|
|
largest = max(float(eigenvalues[-1]), 0.0)
|
|
if largest <= 1e-12:
|
|
slopes = np.zeros(2, dtype=np.float64)
|
|
else:
|
|
supported = np.maximum(eigenvalues, largest * PLANE_VARIANCE_FLOOR_RATIO)
|
|
slopes = eigenvectors @ ((eigenvectors.T @ cross) / supported)
|
|
intercept = value_center - float(center @ slopes)
|
|
return np.asarray((*slopes, intercept), dtype=np.float32)
|
|
|
|
|
|
def depth_balloon_fields(vertices, vertex_uv, depth_data, depth_scale):
|
|
"""Sample camera depth and fit its base plane at a fixed local scale."""
|
|
import numpy as np
|
|
|
|
vertices = np.asarray(vertices, dtype=np.float32)
|
|
uv = np.asarray(vertex_uv, dtype=np.float32)
|
|
if vertices.ndim != 2 or vertices.shape[1] != 3:
|
|
raise ValueError("Depth Balloon vertices must have three components")
|
|
if uv.shape != (len(vertices), 2):
|
|
raise ValueError("Depth Balloon UV coordinates do not match its vertices")
|
|
depth_signal = np.asarray(depth_data["depth_signal"], dtype=np.float32)
|
|
valid = np.asarray(depth_data["valid"], dtype=bool)
|
|
crop_width, crop_height = depth_data["crop_size"]
|
|
depth_height, depth_width = depth_signal.shape
|
|
|
|
crop_x = uv[:, 0] * max(crop_width - 1, 0)
|
|
crop_y = (1.0 - uv[:, 1]) * max(crop_height - 1, 0)
|
|
sample_x = crop_x * max(depth_width - 1, 0) / max(crop_width - 1, 1)
|
|
sample_y = crop_y * max(depth_height - 1, 0) / max(crop_height - 1, 1)
|
|
sampled, usable = _sample_valid_bilinear(
|
|
depth_signal,
|
|
valid,
|
|
sample_x,
|
|
sample_y,
|
|
)
|
|
inside_crop = (
|
|
(crop_x >= 0.0)
|
|
& (crop_x <= max(crop_width - 1, 0))
|
|
& (crop_y >= 0.0)
|
|
& (crop_y <= max(crop_height - 1, 0))
|
|
)
|
|
usable &= inside_crop
|
|
sampled[~usable] = 0.0
|
|
coordinates = np.stack(
|
|
(
|
|
sample_x / max(depth_width - 1, 1) * 2.0 - 1.0,
|
|
sample_y / max(depth_height - 1, 1) * 2.0 - 1.0,
|
|
np.ones(len(vertices), dtype=np.float32),
|
|
),
|
|
axis=1,
|
|
).astype(np.float32)
|
|
|
|
depth_scale = float(depth_scale)
|
|
if not depth_scale > 0.0:
|
|
raise ValueError("Depth Balloon reference scale must be positive")
|
|
normalized_plane = np.asarray(
|
|
depth_data["baseline_vector"],
|
|
dtype=np.float32,
|
|
)
|
|
baseline_values = (
|
|
coordinates @ normalized_plane * depth_scale * 0.5
|
|
)
|
|
local_coordinates = vertices[:, :2]
|
|
if np.count_nonzero(usable) >= 3:
|
|
local_plane = _stable_plane(
|
|
local_coordinates[usable],
|
|
baseline_values[usable],
|
|
)
|
|
else:
|
|
local_plane = np.zeros(3, dtype=np.float32)
|
|
depth_values = sampled * depth_scale
|
|
if usable.any():
|
|
plane_without_offset = local_coordinates @ local_plane[:2]
|
|
residual = depth_values - plane_without_offset
|
|
depth_values[usable] -= float(np.median(residual[usable]))
|
|
return depth_values, local_plane
|
|
|
|
|
|
def orient_depth_balloon_plane(local_plane):
|
|
slope_x, slope_y, offset = (float(value) for value in local_plane)
|
|
return -slope_x, slope_y, offset
|
|
|
|
|
|
def create_depth_balloon_base_plane(context, mesh_object, vertices, local_plane):
|
|
import numpy as np
|
|
from mathutils import Vector
|
|
|
|
vertices = np.asarray(vertices, dtype=np.float32)
|
|
slope_y, slope_z, offset = (float(value) for value in local_plane)
|
|
extent = vertices[:, 1:].max(axis=0) - vertices[:, 1:].min(axis=0)
|
|
center_y = 0.0
|
|
center_z = 0.0
|
|
center_x = offset
|
|
normal = Vector((-1.0, slope_y, slope_z)).normalized()
|
|
|
|
controller = bpy.data.objects.new(f"{mesh_object.name} Base Plane", None)
|
|
context.collection.objects.link(controller)
|
|
controller.parent = mesh_object
|
|
controller.location = (float(center_x), float(center_y), float(center_z))
|
|
controller.rotation_euler = normal.to_track_quat("Z", "Y").to_euler()
|
|
controller.empty_display_type = "SINGLE_ARROW"
|
|
controller.empty_display_size = max(float(extent.max()) * 0.15, 0.01)
|
|
controller.show_in_front = True
|
|
controller.hide_render = True
|
|
controller["anyimage_source_type"] = "DEPTH_BALLOON_BASE_PLANE"
|
|
controller["anyimage_base_plane"] = (slope_y, slope_z, offset)
|
|
return controller
|
|
|
|
|