icrdr/AnyImage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement depth surface strip cleanup and organize tool CLI

Browse Source 
- Added a proposal for depth surface strip cleanup, detailing the rationale and changes to the depth plane and cutout nodes.
- Created specifications for the new cleanup requirements, including scenarios for strip removal and boundary edge detection.
- Developed tasks for implementing shared strip detection logic and integrating new interfaces for depth nodes.
- Established a new CLI structure for organizing node groups and model commands, replacing old entry points with a more intuitive command structure.
- Introduced a prototype for panorama surface generation, including requirements for distance reconstruction and geometry node surfaces.
- Added comprehensive tests for depth strip cleanup functionality, ensuring correct behavior for various mesh topologies and edge cases.
This commit is contained in:
马南
2026-09-10 00:53:43 +08:00
parent 1622afa7a2
commit 27ca25224a
42 changed files with 1970 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
experiments/panorama_surface/geometry.py Normal file
View File

@@ -0,0 +1,147 @@
"""Spherical projection and gradient-domain distance fusion for an isolated trial."""
import numpy as np
from scipy.ndimage import map_coordinates
from scipy.sparse import coo_matrix, diags
from scipy.sparse.linalg import spsolve
def panorama_directions(width, height):
u, v = np.meshgrid((np.arange(width) + 0.5) / width,
(np.arange(height) + 0.5) / height)
return uv_directions(u, v)
def uv_directions(u, v):
theta, phi = 2 * np.pi * u, np.pi * v
return np.stack((np.sin(phi) * np.cos(theta),
np.sin(phi) * np.sin(theta), np.cos(phi)), axis=-1)
def camera_bases():
ratio = (1 + np.sqrt(5)) / 2
vertices = []
for a in (-1, 1):
for b in (-ratio, ratio):
vertices.extend(((0, a, b), (a, b, 0), (b, 0, a)))
bases = []
for forward in np.asarray(vertices):
forward = forward / np.linalg.norm(forward)
right = np.cross(forward, (0, 0, 1))
right /= np.linalg.norm(right)
down = np.cross(forward, right)
bases.append(np.stack((right, down, forward)))
return bases
def view_rays(size):
x, y = np.meshgrid(2 * (np.arange(size) + 0.5) / size - 1,
2 * (np.arange(size) + 0.5) / size - 1)
return np.stack((x, y, np.ones_like(x)), axis=-1)
def sample_panorama(image, directions):
directions = directions / np.linalg.norm(directions, axis=-1, keepdims=True)
u = np.arctan2(directions[..., 1], directions[..., 0]) / (2 * np.pi) % 1
v = np.arccos(np.clip(directions[..., 2], -1, 1)) / np.pi
height, width = image.shape[:2]
padded = np.concatenate((image[:, -1:], image, image[:, :1]), axis=1)
coords = (np.clip(v * height - 0.5, 0, height - 1), u * width + 0.5)
if image.ndim == 2:
return map_coordinates(padded, coords, order=1, mode="nearest")
return np.stack([map_coordinates(padded[..., c], coords, order=1,
mode="nearest") for c in range(image.shape[2])], -1)
def room_reference(directions):
"""Trace a furnished rectangular room from a camera 1.6 m above its floor."""
directions = directions / np.linalg.norm(directions, axis=-1, keepdims=True)
bounds = np.where(directions >= 0, (3.0, 4.0, 1.4), (-3.0, -4.0, -1.6))
hits = bounds / np.where(np.abs(directions) > 1e-12, directions, 1e-12)
hits = np.where(hits > 0, hits, np.inf)
surface = np.argmin(hits, axis=-1)
distance = np.min(hits, axis=-1)
point = directions * distance[..., None]
colors = np.array(((0.68, 0.76, 0.80), (0.81, 0.73, 0.61), (0.9, 0.88, 0.82)))
color = colors[surface].copy()
floor = (surface == 2) & (directions[..., 2] < 0)
tiles = (np.floor(point[..., 0] * 2) + np.floor(point[..., 1] * 2)) % 2
color[floor] = np.stack((0.40 + 0.08 * tiles, 0.27 + 0.06 * tiles,
0.15 + 0.04 * tiles), -1)[floor]
lines = (np.abs(np.sin(point[..., 0] * np.pi * 2)) < 0.04) | (
np.abs(np.sin(point[..., 1] * np.pi * 2)) < 0.04)
color[floor & lines] *= 0.6
window = (surface == 1) & (directions[..., 1] > 0) & (
np.abs(point[..., 0] + 0.4) < 1.05) & (np.abs(point[..., 2] - 0.25) < 0.7)
color[window] = (0.48, 0.72, 0.88)
frame = window & ((np.abs(point[..., 0] + 0.4) < 0.045) |
(np.abs(point[..., 2] - 0.25) < 0.04))
color[frame] = 0.93
for lower, upper, base_color in (
((0.8, 1.0, -1.6), (2.0, 2.2, -0.6), (0.28, 0.43, 0.34)),
((-2.8, -1.8, -1.6), (-2.1, 0.5, 0.7), (0.43, 0.26, 0.15)),
):
safe = np.where(np.abs(directions) > 1e-12, directions, 1e-12)
a, b = np.asarray(lower) / safe, np.asarray(upper) / safe
near, far = np.max(np.minimum(a, b), axis=-1), np.min(np.maximum(a, b), axis=-1)
visible = (near > 0) & (near < far) & (near < distance)
distance[visible] = near[visible]
color[visible] = base_color
color *= (0.75 + 0.25 * np.clip(directions[..., 2] + 0.6, 0, 1))[..., None]
return distance, color
def warp_distances(distances, masks, bases, width, height):
directions = panorama_directions(width, height)
logs, validities = [], []
for distance, mask, basis in zip(distances, masks, bases):
rays = directions @ basis.T
projected = rays[..., :2] / np.maximum(rays[..., 2:], 1e-8)
valid = (rays[..., 2] > 0) & (np.abs(projected) < 0.98).all(axis=-1)
size = distance.shape[0]
coords = ((projected[..., 1] + 1) * size / 2 - 0.5,
(projected[..., 0] + 1) * size / 2 - 0.5)
valid &= map_coordinates(mask.astype(float), coords, order=0, mode="nearest") > 0.5
logs.append(map_coordinates(np.log(np.maximum(distance, 1e-6)), coords,
order=1, mode="nearest"))
validities.append(valid)
return np.asarray(logs), np.asarray(validities)
def merge_distances(distances, masks, bases, width, height):
"""Fuse log-distance derivatives with a cyclic longitude and a scale anchor.
Algorithm reference: microsoft/MoGe, moge/utils/panorama.py.
This independent implementation solves normal equations on a small trial grid.
"""
logs, valid = warp_distances(distances, masks, bases, width, height)
count = width * height
index = np.arange(count).reshape(height, width)
start = np.concatenate((index.ravel(), index[:-1].ravel()))
end = np.concatenate((np.roll(index, -1, axis=1).ravel(), index[1:].ravel()))
rows = np.arange(len(start))
gradient = coo_matrix((np.concatenate((-np.ones(len(start)), np.ones(len(start)))),
(np.tile(rows, 2), np.concatenate((start, end)))),
shape=(len(start), count)).tocsr()
laplacian = -(gradient.T @ gradient).tocsr()
edge_valid = valid.reshape(12, -1)[:, start] & valid.reshape(12, -1)[:, end]
grad_samples = logs.reshape(12, -1)[:, end] - logs.reshape(12, -1)[:, start]
grad_target = np.sum(grad_samples * edge_valid, axis=0) / np.maximum(edge_valid.sum(0), 1)
lap_valid = valid.copy()
lap_valid &= np.roll(valid, 1, axis=2) & np.roll(valid, -1, axis=2)
lap_valid[:, 1:] &= valid[:, :-1]
lap_valid[:, :-1] &= valid[:, 1:]
lap_valid = lap_valid.reshape(12, -1)
lap_samples = np.asarray((laplacian @ logs.reshape(12, -1).T).T)
lap_target = np.sum(lap_samples * lap_valid, axis=0) / np.maximum(lap_valid.sum(0), 1)
g = gradient[edge_valid.any(0)]
l = laplacian[lap_valid.any(0)]
coverage = valid.any(0)
if not coverage.any():
raise ValueError("No valid panorama predictions")
baseline = np.sum(logs * valid, axis=0) / np.maximum(valid.sum(0), 1)
matrix = g.T @ g + l.T @ l + diags(np.full(count, 1e-7))
target = g.T @ grad_target[edge_valid.any(0)] + l.T @ lap_target[lap_valid.any(0)]
solution = spsolve(matrix.tocsc(), target + 1e-7 * baseline.ravel()).reshape(height, width)
solution += np.median(baseline[coverage] - solution[coverage])
return np.exp(solution).astype(np.float32), coverage