icrdr/AnyImage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement depth surface strip cleanup and organize tool CLI

Browse Source 
- Added a proposal for depth surface strip cleanup, detailing the rationale and changes to the depth plane and cutout nodes.
- Created specifications for the new cleanup requirements, including scenarios for strip removal and boundary edge detection.
- Developed tasks for implementing shared strip detection logic and integrating new interfaces for depth nodes.
- Established a new CLI structure for organizing node groups and model commands, replacing old entry points with a more intuitive command structure.
- Introduced a prototype for panorama surface generation, including requirements for distance reconstruction and geometry node surfaces.
- Added comprehensive tests for depth strip cleanup functionality, ensuring correct behavior for various mesh topologies and edge cases.
This commit is contained in:
马南
2026-09-10 00:53:43 +08:00
parent 1622afa7a2
commit 27ca25224a
42 changed files with 1970 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
experiments/panorama_surface/.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
/output/
/analytic/
/photo/

5
experiments/panorama_surface/ATTRIBUTION.md Normal file
View File

@@ -0,0 +1,5 @@
# 素材来源
真实照片试验使用 DerMische 的 [Braunschweig Sankt-Ägidien Panorama 02](https://commons.wikimedia.org/wiki/File:Braunschweig_Sankt-%C3%84gidien_Panorama_02.jpg)，许可为 [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)。使用 Wikimedia 的 1280×640 缩略图作为输入，并通过 MoGe2 估计距离、重投影和生成网格。照片及相应衍生预览、带照片纹理的网格遵循该许可，作者不为本试验背书。
多视图投影与梯度域距离融合参考 [microsoft/MoGe 的 panorama.py](https://github.com/microsoft/MoGe/blob/main/moge/utils/panorama.py)；固定视角几何恢复参考 [MoGe v2](https://github.com/microsoft/MoGe/blob/main/moge/model/v2.py)。试验独立实现这些计算。

185
experiments/panorama_surface/blender_scene.py Normal file
View File

@@ -0,0 +1,185 @@
"""Build or verify a standalone panorama Geometry Nodes demonstration offline."""
import argparse
from pathlib import Path
import sys
import bpy
import numpy as np
from mathutils import Vector
sys.path.insert(0, str(Path(__file__).resolve().parent))
from quad_surface import build_node_group
from validation import evaluated_points, socket_id, verify
def create_distance_image(name, path, output_path):
with np.load(path) as data:
distance, alpha = data["distance"], data["alpha"]
height, width = distance.shape
image = bpy.data.images.new(name, width=width, height=height, alpha=True, float_buffer=True)
image.colorspace_settings.name = "Non-Color"
image.alpha_mode = "CHANNEL_PACKED"
rgba = np.concatenate((np.repeat(distance[..., None], 3, -1), alpha[..., None]), -1)
image.pixels.foreach_set(rgba[::-1].astype(np.float32).ravel())
image.filepath_raw = str(output_path)
image.file_format = "OPEN_EXR"
image.save()
image.pack()
return image
def create_material(image):
material = bpy.data.materials.new("Panorama Color")
material.use_nodes = True
material.node_tree.nodes.clear()
nodes, links = material.node_tree.nodes, material.node_tree.links
uv = nodes.new("ShaderNodeUVMap")
uv.uv_map = "UVMap"
uv.location = (0, 0)
texture = nodes.new("ShaderNodeTexImage")
texture.image = image
texture.extension = "REPEAT"
texture.location = (200, 0)
emission = nodes.new("ShaderNodeEmission")
emission.location = (500, 0)
output = nodes.new("ShaderNodeOutputMaterial")
output.location = (700, 0)
links.new(uv.outputs[0], texture.inputs[0])
links.new(texture.outputs[0], emission.inputs[0])
links.new(emission.outputs[0], output.inputs[0])
return material
def build_scene(output):
bpy.ops.wm.read_factory_settings(use_empty=True)
group = build_node_group()
color = bpy.data.images.load(str(output / "panorama.png"))
color.pack()
material = create_material(color)
objects = []
for name, filename, offset in (("Reference Room", "reference.npz", -4),
("MoGe2 Panorama", "prediction.npz", 4)):
if not (output / filename).exists():
continue
depth_path = output / ("reference_depth.exr" if filename == "reference.npz" else "depth.exr")
distance = create_distance_image(depth_path.name, output / filename, depth_path)
mesh = bpy.data.meshes.new("Panorama Host")
obj = bpy.data.objects.new(name, mesh)
bpy.context.collection.objects.link(obj)
obj.location.x = offset
obj.data.materials.append(material)
modifier = obj.modifiers.new("Panorama Surface", "NODES")
modifier.node_group = group
modifier[socket_id(group, "Distance")] = distance
modifier[socket_id(group, "Subdivide")] = 7
modifier[socket_id(group, "Depth Scale")] = 1.0
modifier[socket_id(group, "Validity Threshold")] = 0.9
modifier[socket_id(group, "Material")] = material
objects.append(obj)
for obj in objects:
obj.select_set(False)
objects[-1].select_set(True)
bpy.context.view_layer.objects.active = objects[-1]
scene = bpy.context.scene
scene.render.engine = "CYCLES"
scene.cycles.samples = 16
scene.render.resolution_x, scene.render.resolution_y = 1400, 750
scene.render.resolution_percentage = 100
scene.world = bpy.data.worlds.new("Trial World")
scene.world.color = (0.18, 0.18, 0.18)
scene.view_settings.view_transform = "Standard"
camera_data = bpy.data.cameras.new("Overview")
camera = bpy.data.objects.new("Overview", camera_data)
bpy.context.collection.objects.link(camera)
camera.location = (11, -17, 13)
camera.rotation_euler = (Vector((0, 0, 0)) - camera.location).to_track_quat("-Z", "Y").to_euler()
camera_data.type, camera_data.ortho_scale = "ORTHO", 20
if len(objects) == 1:
points, _ = evaluated_points(objects[0])
points += np.array(objects[0].location)
center = (points.min(0) + points.max(0)) / 2
extent = float(np.max(np.ptp(points, axis=0)))
camera.location = Vector(center) + Vector((0.6, -1, 1)) * extent
camera.rotation_euler = (Vector(center) - camera.location).to_track_quat("-Z", "Y").to_euler()
camera_data.ortho_scale = extent * 1.8
scene.camera = camera
capture_camera = None
if len(objects) == 1:
capture_data = bpy.data.cameras.new("Capture Center")
capture_data.lens = 22
capture_camera = bpy.data.objects.new("Capture Center", capture_data)
bpy.context.collection.objects.link(capture_camera)
capture_camera.location = objects[0].location
capture_camera.rotation_euler = Vector((-1, 0, 0.15)).to_track_quat("-Z", "Y").to_euler()
bpy.data.texts.new("Attribution").write(
(Path(__file__).parent / "ATTRIBUTION.md").read_text(encoding="utf-8"))
bpy.data.texts.new("试验说明").write(
"Subdivide 控制 Quad Sphere 细分，全有效面数为 6 × 4^Subdivide。\n"
"Depth Scale：0 为单位球面，1 为全景距离重建。\n"
"深度 EXR 的 Alpha 已包含 Mask × 源图 Alpha，低于 Validity Threshold 的区域会被剔除。\n"
"Quad Sphere 来自 O_Essentials，节点组和依赖已打包。\n"
)
for screen in bpy.data.screens:
for area in screen.areas:
if area.type == "VIEW_3D":
area.spaces.active.shading.type = "MATERIAL"
area.spaces.active.region_3d.view_distance = extent * 1.4 if len(objects) == 1 else 18
if len(objects) == 1:
area.spaces.active.region_3d.view_location = Vector(center)
area.spaces.active.region_3d.view_rotation = camera.rotation_euler.to_quaternion()
verify(output)
bpy.ops.wm.save_as_mainfile(filepath=str(output / "panorama_trial.blend"))
# The render uses a temporary inspection cut; the saved asset retains the full shell.
for obj in objects:
if len(objects) == 1:
continue
modifier = obj.modifiers.new("Inspection Cut", "NODES")
cut_group = bpy.data.node_groups.new(f"{obj.name} Inspection", "GeometryNodeTree")
cut_group.interface.new_socket(name="Geometry", in_out="INPUT", socket_type="NodeSocketGeometry")
cut_group.interface.new_socket(name="Geometry", in_out="OUTPUT", socket_type="NodeSocketGeometry")
source = cut_group.nodes.new("NodeGroupInput")
position = cut_group.nodes.new("GeometryNodeInputPosition")
xyz = cut_group.nodes.new("ShaderNodeSeparateXYZ")
above = cut_group.nodes.new("ShaderNodeMath")
above.operation = "GREATER_THAN"
above.inputs[1].default_value = 0.4
front = cut_group.nodes.new("ShaderNodeMath")
front.operation = "LESS_THAN"
front.inputs[1].default_value = -0.7
either = cut_group.nodes.new("ShaderNodeMath")
either.operation = "MAXIMUM"
remove = cut_group.nodes.new("GeometryNodeDeleteGeometry")
remove.domain = "FACE"
result = cut_group.nodes.new("NodeGroupOutput")
for a, b in ((position.outputs[0], xyz.inputs[0]),
(xyz.outputs["Z"], above.inputs[0]), (xyz.outputs["Y"], front.inputs[0]),
(above.outputs[0], either.inputs[0]), (front.outputs[0], either.inputs[1]),
(source.outputs[0], remove.inputs["Geometry"]),
(either.outputs[0], remove.inputs["Selection"]),
(remove.outputs[0], result.inputs[0])):
cut_group.links.new(a, b)
modifier.node_group = cut_group
scene.render.filepath = str(output / "comparison.png")
bpy.ops.render.render(write_still=True)
if capture_camera is not None:
scene.camera = capture_camera
scene.render.filepath = str(output / "capture_view.png")
bpy.ops.render.render(write_still=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--verify", action="store_true")
args = parser.parse_args(sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else [])
output = args.output.resolve()
if args.verify:
bpy.ops.wm.open_mainfile(filepath=str(output / "panorama_trial.blend"))
verify(output)
else:
build_scene(output)
if __name__ == "__main__":
main()

147
experiments/panorama_surface/geometry.py Normal file
View File

@@ -0,0 +1,147 @@
"""Spherical projection and gradient-domain distance fusion for an isolated trial."""
import numpy as np
from scipy.ndimage import map_coordinates
from scipy.sparse import coo_matrix, diags
from scipy.sparse.linalg import spsolve
def panorama_directions(width, height):
u, v = np.meshgrid((np.arange(width) + 0.5) / width,
(np.arange(height) + 0.5) / height)
return uv_directions(u, v)
def uv_directions(u, v):
theta, phi = 2 * np.pi * u, np.pi * v
return np.stack((np.sin(phi) * np.cos(theta),
np.sin(phi) * np.sin(theta), np.cos(phi)), axis=-1)
def camera_bases():
ratio = (1 + np.sqrt(5)) / 2
vertices = []
for a in (-1, 1):
for b in (-ratio, ratio):
vertices.extend(((0, a, b), (a, b, 0), (b, 0, a)))
bases = []
for forward in np.asarray(vertices):
forward = forward / np.linalg.norm(forward)
right = np.cross(forward, (0, 0, 1))
right /= np.linalg.norm(right)
down = np.cross(forward, right)
bases.append(np.stack((right, down, forward)))
return bases
def view_rays(size):
x, y = np.meshgrid(2 * (np.arange(size) + 0.5) / size - 1,
2 * (np.arange(size) + 0.5) / size - 1)
return np.stack((x, y, np.ones_like(x)), axis=-1)
def sample_panorama(image, directions):
directions = directions / np.linalg.norm(directions, axis=-1, keepdims=True)
u = np.arctan2(directions[..., 1], directions[..., 0]) / (2 * np.pi) % 1
v = np.arccos(np.clip(directions[..., 2], -1, 1)) / np.pi
height, width = image.shape[:2]
padded = np.concatenate((image[:, -1:], image, image[:, :1]), axis=1)
coords = (np.clip(v * height - 0.5, 0, height - 1), u * width + 0.5)
if image.ndim == 2:
return map_coordinates(padded, coords, order=1, mode="nearest")
return np.stack([map_coordinates(padded[..., c], coords, order=1,
mode="nearest") for c in range(image.shape[2])], -1)
def room_reference(directions):
"""Trace a furnished rectangular room from a camera 1.6 m above its floor."""
directions = directions / np.linalg.norm(directions, axis=-1, keepdims=True)
bounds = np.where(directions >= 0, (3.0, 4.0, 1.4), (-3.0, -4.0, -1.6))
hits = bounds / np.where(np.abs(directions) > 1e-12, directions, 1e-12)
hits = np.where(hits > 0, hits, np.inf)
surface = np.argmin(hits, axis=-1)
distance = np.min(hits, axis=-1)
point = directions * distance[..., None]
colors = np.array(((0.68, 0.76, 0.80), (0.81, 0.73, 0.61), (0.9, 0.88, 0.82)))
color = colors[surface].copy()
floor = (surface == 2) & (directions[..., 2] < 0)
tiles = (np.floor(point[..., 0] * 2) + np.floor(point[..., 1] * 2)) % 2
color[floor] = np.stack((0.40 + 0.08 * tiles, 0.27 + 0.06 * tiles,
0.15 + 0.04 * tiles), -1)[floor]
lines = (np.abs(np.sin(point[..., 0] * np.pi * 2)) < 0.04) | (
np.abs(np.sin(point[..., 1] * np.pi * 2)) < 0.04)
color[floor & lines] *= 0.6
window = (surface == 1) & (directions[..., 1] > 0) & (
np.abs(point[..., 0] + 0.4) < 1.05) & (np.abs(point[..., 2] - 0.25) < 0.7)
color[window] = (0.48, 0.72, 0.88)
frame = window & ((np.abs(point[..., 0] + 0.4) < 0.045) |
(np.abs(point[..., 2] - 0.25) < 0.04))
color[frame] = 0.93
for lower, upper, base_color in (
((0.8, 1.0, -1.6), (2.0, 2.2, -0.6), (0.28, 0.43, 0.34)),
((-2.8, -1.8, -1.6), (-2.1, 0.5, 0.7), (0.43, 0.26, 0.15)),
):
safe = np.where(np.abs(directions) > 1e-12, directions, 1e-12)
a, b = np.asarray(lower) / safe, np.asarray(upper) / safe
near, far = np.max(np.minimum(a, b), axis=-1), np.min(np.maximum(a, b), axis=-1)
visible = (near > 0) & (near < far) & (near < distance)
distance[visible] = near[visible]
color[visible] = base_color
color *= (0.75 + 0.25 * np.clip(directions[..., 2] + 0.6, 0, 1))[..., None]
return distance, color
def warp_distances(distances, masks, bases, width, height):
directions = panorama_directions(width, height)
logs, validities = [], []
for distance, mask, basis in zip(distances, masks, bases):
rays = directions @ basis.T
projected = rays[..., :2] / np.maximum(rays[..., 2:], 1e-8)
valid = (rays[..., 2] > 0) & (np.abs(projected) < 0.98).all(axis=-1)
size = distance.shape[0]
coords = ((projected[..., 1] + 1) * size / 2 - 0.5,
(projected[..., 0] + 1) * size / 2 - 0.5)
valid &= map_coordinates(mask.astype(float), coords, order=0, mode="nearest") > 0.5
logs.append(map_coordinates(np.log(np.maximum(distance, 1e-6)), coords,
order=1, mode="nearest"))
validities.append(valid)
return np.asarray(logs), np.asarray(validities)
def merge_distances(distances, masks, bases, width, height):
"""Fuse log-distance derivatives with a cyclic longitude and a scale anchor.
Algorithm reference: microsoft/MoGe, moge/utils/panorama.py.
This independent implementation solves normal equations on a small trial grid.
"""
logs, valid = warp_distances(distances, masks, bases, width, height)
count = width * height
index = np.arange(count).reshape(height, width)
start = np.concatenate((index.ravel(), index[:-1].ravel()))
end = np.concatenate((np.roll(index, -1, axis=1).ravel(), index[1:].ravel()))
rows = np.arange(len(start))
gradient = coo_matrix((np.concatenate((-np.ones(len(start)), np.ones(len(start)))),
(np.tile(rows, 2), np.concatenate((start, end)))),
shape=(len(start), count)).tocsr()
laplacian = -(gradient.T @ gradient).tocsr()
edge_valid = valid.reshape(12, -1)[:, start] & valid.reshape(12, -1)[:, end]
grad_samples = logs.reshape(12, -1)[:, end] - logs.reshape(12, -1)[:, start]
grad_target = np.sum(grad_samples * edge_valid, axis=0) / np.maximum(edge_valid.sum(0), 1)
lap_valid = valid.copy()
lap_valid &= np.roll(valid, 1, axis=2) & np.roll(valid, -1, axis=2)
lap_valid[:, 1:] &= valid[:, :-1]
lap_valid[:, :-1] &= valid[:, 1:]
lap_valid = lap_valid.reshape(12, -1)
lap_samples = np.asarray((laplacian @ logs.reshape(12, -1).T).T)
lap_target = np.sum(lap_samples * lap_valid, axis=0) / np.maximum(lap_valid.sum(0), 1)
g = gradient[edge_valid.any(0)]
l = laplacian[lap_valid.any(0)]
coverage = valid.any(0)
if not coverage.any():
raise ValueError("No valid panorama predictions")
baseline = np.sum(logs * valid, axis=0) / np.maximum(valid.sum(0), 1)
matrix = g.T @ g + l.T @ l + diags(np.full(count, 1e-7))
target = g.T @ grad_target[edge_valid.any(0)] + l.T @ lap_target[lap_valid.any(0)]
solution = spsolve(matrix.tocsc(), target + 1e-7 * baseline.ravel()).reshape(height, width)
solution += np.median(baseline[coverage] - solution[coverage])
return np.exp(solution).astype(np.float32), coverage

144
experiments/panorama_surface/prototype.py Normal file
View File

@@ -0,0 +1,144 @@
"""Run MoGe2 panorama inference without importing or changing the add-on."""
import argparse
import json
from pathlib import Path
import sys
import time
import numpy as np
from PIL import Image
from scipy.optimize import least_squares
from geometry import (camera_bases, merge_distances, panorama_directions,
room_reference, sample_panorama, view_rays)
def fixed_fov_distance(points, mask, metric_scale):
"""Recover camera Z shift using the known 90-degree perspective rays."""
size = points.shape[0]
rays = view_rays(size)
select = mask > 0.5
select &= np.isfinite(points).all(-1)
if select.sum() < 16:
return np.ones((size, size), dtype=np.float32), np.zeros((size, size), dtype=bool)
stride = max(1, size // 64)
sampled = select[::stride, ::stride]
p = points[::stride, ::stride][sampled].astype(float)
q = rays[::stride, ::stride, :2][sampled]
initial = np.sum(q * (p[:, :2] - q * p[:, 2:])) / max(np.sum(q * q), 1e-8)
lower = -float(points[..., 2][select].min()) + 1e-5
fitted = least_squares(lambda shift: (p[:, :2] / (p[:, 2:] + shift[0]) - q).ravel(),
[max(initial, lower + 1e-4)], bounds=([lower], [np.inf]))
depth = (points[..., 2] + fitted.x[0]) * float(metric_scale)
select &= (depth > 0) & np.isfinite(depth)
distance = np.maximum(depth, 1e-6) * np.linalg.norm(rays, axis=-1)
return distance.astype(np.float32), select
def depth_alpha(image, validity):
"""Combine the model mask with nearest-resized source alpha once at export."""
height, width = validity.shape
if image.shape[-1] == 4:
source = Image.fromarray(image[..., 3].astype(np.float32))
alpha = np.asarray(source.resize((width, height), Image.Resampling.NEAREST))
else:
alpha = np.ones((height, width), dtype=np.float32)
return np.asarray(validity, dtype=np.float32) * alpha
def remove_background(image):
"""Use the project's existing BEN2 implementation for the trial input."""
root = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(root / "src/anyimage/server/models"))
from onnx_ben2 import create_session, infer
session = create_session(root / "models/BEN2-ONNX", "cpu")
return infer(session, image, release_memory=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--image", type=Path)
parser.add_argument("--model", type=Path, default=Path("models/moge-2-vits-normal-onnx/model.onnx"))
parser.add_argument("--output", type=Path, default=Path(__file__).parent / "output")
parser.add_argument("--size", type=int, default=256)
parser.add_argument("--width", type=int, default=256)
parser.add_argument("--reference-only", action="store_true")
parser.add_argument("--remove-background", action="store_true")
args = parser.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
width, height = args.width, args.width // 2
directions = panorama_directions(width, height)
truth, _ = room_reference(directions)
if args.image:
with Image.open(args.image) as source:
source = source.convert("RGBA")
if args.remove_background:
foreground = remove_background(source)
original_alpha = np.asarray(source.getchannel("A"), dtype=np.float32) / 255
alpha = np.asarray(foreground.getchannel("A"), dtype=np.float32) * original_alpha
source.putalpha(Image.fromarray(alpha.astype(np.uint8)))
image = np.asarray(source, dtype=np.float32) / 255
if image.shape[1] != 2 * image.shape[0]:
raise ValueError("Expected a full 2:1 equirectangular panorama")
else:
_, image = room_reference(panorama_directions(1024, 512))
np.savez_compressed(args.output / "reference.npz", distance=truth.astype(np.float32),
alpha=np.ones_like(truth, dtype=np.float32))
Image.fromarray(np.uint8(np.clip(image, 0, 1) * 255)).save(args.output / "panorama.png")
bases = camera_bases()
distances, masks, panels = [], [], []
start = time.perf_counter()
if not args.reference_only:
import onnxruntime as ort
options = ort.SessionOptions()
options.intra_op_num_threads = 8
session = ort.InferenceSession(str(args.model), options, providers=["CPUExecutionProvider"])
scale_name = next(item.name for item in session.get_outputs()
if item.name in {"scale", "metric_scale"})
for index, basis in enumerate(bases):
rays = view_rays(args.size) @ basis
view = sample_panorama(image, rays).astype(np.float32)
source_valid = np.ones(view.shape[:2], dtype=bool)
if view.shape[-1] == 4:
source_valid = view[..., 3] > 0
view = view[..., :3] * view[..., 3:]
panels.append(np.uint8(np.clip(view, 0, 1) * 255))
if args.reference_only:
distance, _ = room_reference(rays)
mask = np.ones_like(distance, dtype=bool)
else:
prediction = session.run(["points", "mask", scale_name],
{"image": view.transpose(2, 0, 1)[None],
"num_tokens": np.array(1200, dtype=np.int64)})
distance, mask = fixed_fov_distance(prediction[0][0], prediction[1][0] * source_valid,
prediction[2].reshape(-1)[0])
mask &= source_valid
distances.append(distance)
masks.append(mask)
print(f"View {index + 1}/12: valid={mask.mean():.3f}, elapsed={time.perf_counter()-start:.1f}s", flush=True)
Image.fromarray(np.concatenate([np.concatenate(panels[i:i + 4], axis=1)
for i in range(0, 12, 4)], axis=0)).save(args.output / "views.png")
np.savez_compressed(args.output / "views.npz", distances=distances, masks=masks)
distance, validity = merge_distances(distances, masks, bases, width, height)
combined_alpha = depth_alpha(image, validity)
np.savez_compressed(args.output / "prediction.npz", distance=distance, alpha=combined_alpha)
stats = {"mode": "analytic" if args.reference_only else args.model.parent.name + "-cpu",
"input": str(args.image) if args.image else "synthetic room",
"background_removal": "BEN2" if args.remove_background else None,
"size": [width, height], "valid_fraction": float(validity.mean()),
"depth_alpha_valid_fraction": float((combined_alpha >= 0.9).mean()),
"seconds": time.perf_counter() - start,
"distance_range": [float(distance.min()), float(distance.max())]}
if not args.image:
ratio = np.median(truth[validity] / distance[validity])
stats["reference_scale_alignment"] = float(ratio)
stats["aligned_mean_relative_error"] = float(np.mean(np.abs(distance[validity] * ratio / truth[validity] - 1)))
stats["aligned_seam_mean_relative_error"] = float(np.mean(np.abs(distance[:, [0, -1]] * ratio / truth[:, [0, -1]] - 1)))
(args.output / "inference.json").write_text(json.dumps(stats, indent=2), encoding="utf-8")
print(json.dumps(stats, indent=2), flush=True)
if __name__ == "__main__":
main()

151
experiments/panorama_surface/quad_surface.py Normal file
View File

@@ -0,0 +1,151 @@
"""Build an offline Quad Sphere panorama asset with point-domain validity culling."""
from pathlib import Path
import sys
import bpy
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from tools.nodes.common.nodes import interface_socket, prepare_node_group
REFERENCE_LIBRARY = Path("D:/Projects/OmooNodes/subspaces/O_Essentials/O_Essentials_GeometryNodes.blend1")
def build_node_group():
with bpy.data.libraries.load(str(REFERENCE_LIBRARY), link=False) as (source, target):
if "O Quad Sphere" not in source.node_groups:
raise ValueError("Reference library has no O Quad Sphere")
target.node_groups = ["O Quad Sphere"]
group = bpy.data.node_groups.new("Panorama Surface Trial", "GeometryNodeTree")
group.is_modifier = True
for name, kind in (("Subdivide", "NodeSocketInt"), ("Depth Scale", "NodeSocketFloat"),
("Validity Threshold", "NodeSocketFloat"),
("Distance", "NodeSocketImage"), ("Material", "NodeSocketMaterial")):
socket = interface_socket(group, name, "INPUT", kind)
if name == "Subdivide":
socket.default_value, socket.min_value, socket.max_value = 7, 0, 8
elif name == "Depth Scale":
socket.default_value, socket.min_value, socket.max_value = 1.0, 0.0, 1.0
socket.subtype = "FACTOR"
elif name == "Validity Threshold":
socket.default_value, socket.min_value, socket.max_value = 0.9, 0.0, 1.0
socket.subtype = "FACTOR"
elif name == "Material":
socket.hide_in_modifier = True
interface_socket(group, "Geometry", "OUTPUT", "NodeSocketGeometry")
links = group.links
def node(kind, x, y):
result = group.nodes.new(kind)
result.location = (x, y)
return result
def math(operation, a, b, x, y):
result = node("ShaderNodeMath", x, y)
result.operation = operation
for index, value in enumerate((a, b)):
if isinstance(value, (float, int)):
result.inputs[index].default_value = value
else:
links.new(value, result.inputs[index])
return result.outputs[0]
def store(geometry, name, field, domain, x, y):
result = node("GeometryNodeStoreNamedAttribute", x, y)
result.data_type, result.domain = "FLOAT_VECTOR", domain
result.inputs["Name"].default_value = name
links.new(geometry, result.inputs["Geometry"])
links.new(field, result.inputs["Value"])
return result.outputs["Geometry"]
def read(name, x, y):
result = node("GeometryNodeInputNamedAttribute", x, y)
result.data_type = "FLOAT_VECTOR"
result.inputs["Name"].default_value = name
return result.outputs["Attribute"]
inputs = node("NodeGroupInput", 0, 400)
segments = math("POWER", 2, inputs.outputs["Subdivide"], 0, 150)
segments = math("ADD", segments, 1, 180, 150)
sphere = node("GeometryNodeGroup", 400, 400)
sphere.node_tree = target.node_groups[0]
sphere.inputs["Radius"].default_value = 1
links.new(segments, sphere.inputs["Segments"])
position = node("GeometryNodeInputPosition", 400, 100)
geometry = store(sphere.outputs["Mesh"], "panorama_direction", position.outputs[0], "POINT", 620, 400)
xyz = node("ShaderNodeSeparateXYZ", 620, 100)
links.new(read("panorama_direction", 400, -80), xyz.inputs[0])
angle = math("ARCTAN2", xyz.outputs["Y"], xyz.outputs["X"], 800, 100)
longitude = math("DIVIDE", angle, 6.283185307179586, 980, 100)
longitude = math("FRACT", math("ADD", longitude, 1, 1160, 100), 0, 1340, 100)
latitude = math("ARCCOSINE", xyz.outputs["Z"], 0, 800, -100)
latitude = math("SUBTRACT", 1, math("DIVIDE", latitude, 3.141592653589793, 980, -100), 1160, -100)
uv = node("ShaderNodeCombineXYZ", 1540, 100)
links.new(longitude, uv.inputs["X"])
links.new(latitude, uv.inputs["Y"])
geometry = store(geometry, "panorama_sample_uv", uv.outputs[0], "POINT", 1540, 400)
# Each face chooses its own longitude branch; pole corners use the face longitude.
face_position = node("GeometryNodeFieldOnDomain", 800, -400)
face_position.data_type, face_position.domain = "FLOAT_VECTOR", "FACE"
links.new(position.outputs[0], face_position.inputs["Value"])
face_xyz = node("ShaderNodeSeparateXYZ", 980, -400)
links.new(face_position.outputs[0], face_xyz.inputs[0])
face_angle = math("ARCTAN2", face_xyz.outputs["Y"], face_xyz.outputs["X"], 1160, -400)
face_u = math("DIVIDE", face_angle, 6.283185307179586, 1340, -400)
correction = math("ROUND", math("SUBTRACT", face_u, longitude, 1540, -300), 0, 1720, -300)
corner_u = math("ADD", longitude, correction, 1900, -300)
pole = math("GREATER_THAN", math("ABSOLUTE", xyz.outputs["Z"], 0, 1540, -520), 0.999999, 1720, -520)
choose_u = node("GeometryNodeSwitch", 2080, -300)
choose_u.input_type = "FLOAT"
links.new(pole, choose_u.inputs["Switch"])
links.new(corner_u, choose_u.inputs["False"])
links.new(face_u, choose_u.inputs["True"])
corner_uv = node("ShaderNodeCombineXYZ", 2300, -300)
links.new(choose_u.outputs[0], corner_uv.inputs["X"])
links.new(latitude, corner_uv.inputs["Y"])
geometry = store(geometry, "UVMap", corner_uv.outputs[0], "CORNER", 2300, 400)
samples = node("NodeGroupInput", 2520, -150)
sample_uv = read("panorama_sample_uv", 2520, -400)
texture = node("GeometryNodeImageTexture", 2740, 0)
texture.interpolation, texture.extension = "Closest", "EXTEND"
links.new(samples.outputs["Distance"], texture.inputs["Image"])
links.new(sample_uv, texture.inputs["Vector"])
depth = node("ShaderNodeSeparateXYZ", 2960, 0)
links.new(texture.outputs["Color"], depth.inputs[0])
blend = node("ShaderNodeMapRange", 3180, 0)
blend.clamp = True
blend.inputs["To Min"].default_value = 1
links.new(inputs.outputs["Depth Scale"], blend.inputs["Value"])
links.new(depth.outputs["X"], blend.inputs["To Max"])
radial = node("ShaderNodeVectorMath", 3400, 0)
radial.operation = "SCALE"
links.new(read("panorama_direction", 3180, -260), radial.inputs[0])
links.new(blend.outputs["Result"], radial.inputs[3])
deform = node("GeometryNodeSetPosition", 3620, 400)
links.new(geometry, deform.inputs["Geometry"])
links.new(radial.outputs[0], deform.inputs["Position"])
channels = node("FunctionNodeSeparateColor", 2960, -430)
channels.mode = "RGB"
links.new(texture.outputs["Color"], channels.inputs["Color"])
invalid = node("FunctionNodeCompare", 3400, -500)
invalid.data_type, invalid.operation = "FLOAT", "LESS_THAN"
links.new(channels.outputs["Alpha"], invalid.inputs["A"])
links.new(inputs.outputs["Validity Threshold"], invalid.inputs["B"])
point_invalid = node("GeometryNodeFieldOnDomain", 3620, -500)
point_invalid.data_type, point_invalid.domain = "BOOLEAN", "POINT"
links.new(invalid.outputs["Result"], point_invalid.inputs["Value"])
delete = node("GeometryNodeDeleteGeometry", 3840, 400)
delete.domain, delete.mode = "FACE", "ALL"
links.new(deform.outputs["Geometry"], delete.inputs["Geometry"])
links.new(point_invalid.outputs[0], delete.inputs["Selection"])
material = node("GeometryNodeSetMaterial", 4060, 400)
links.new(delete.outputs["Geometry"], material.inputs["Geometry"])
links.new(inputs.outputs["Material"], material.inputs["Material"])
output = node("NodeGroupOutput", 4280, 400)
links.new(material.outputs["Geometry"], output.inputs[0])
prepare_node_group(group)
return group

93
experiments/panorama_surface/test_geometry.py Normal file
View File

@@ -0,0 +1,93 @@
"""Behavior checks with independent spherical and room geometry references."""
import numpy as np
from geometry import (camera_bases, merge_distances, panorama_directions,
room_reference, sample_panorama, view_rays)
from prototype import fixed_fov_distance
def test_fixed_fov_recovers_shift_and_metric_distance():
rays = view_rays(64)
depth = 2 + 0.3 * rays[..., 0]
points = rays * depth[..., None]
points[..., 2] -= 0.7
distance, mask = fixed_fov_distance(points, np.ones((64, 64)), 1.5)
np.testing.assert_allclose(distance, depth * np.linalg.norm(rays, axis=-1) * 1.5, rtol=1e-5)
assert mask.all()
def test_longitude_sampling_wraps_without_a_seam():
directions = panorama_directions(256, 128)
image = 2 + directions[..., 0]
rays = np.array(((1, 1e-8, 0), (1, -1e-8, 0)))
sampled = sample_panorama(image, rays)
np.testing.assert_allclose(sampled[0], sampled[1], atol=1e-6)
np.testing.assert_allclose(sampled, 3, atol=0.001)
def test_fusion_removes_per_view_scale_offsets():
bases = camera_bases()
scales = np.exp(np.linspace(-0.4, 0.4, 12))
distances = []
for scale, basis in zip(scales, bases):
rays = view_rays(128) @ basis
rays /= np.linalg.norm(rays, axis=-1, keepdims=True)
distances.append((3 + 0.4 * rays[..., 0] + 0.2 * rays[..., 2]) * scale)
merged, mask = merge_distances(distances, np.ones((12, 128, 128), bool), bases, 64, 32)
rays = panorama_directions(64, 32)
expected = 3 + 0.4 * rays[..., 0] + 0.2 * rays[..., 2]
merged *= np.median(expected / merged)
assert mask.all()
assert np.mean(np.abs(merged / expected - 1)) < 0.002
assert np.max(np.abs(merged[:, [0, -1]] / expected[:, [0, -1]] - 1)) < 0.005
def test_room_distance_lands_on_floor():
rays = np.array(((0.0, 0.0, -1.0), (0.2, -0.3, -1.0)))
rays /= np.linalg.norm(rays, axis=-1, keepdims=True)
distance, _ = room_reference(rays)
np.testing.assert_allclose((rays * distance[:, None])[:, 2], -1.6)
def test_sky_view_returns_invalid_samples():
distance, mask = fixed_fov_distance(np.ones((32, 32, 3)), np.zeros((32, 32)), 1)
assert not mask.any()
assert np.isfinite(distance).all()
def test_room_projection_fusion_preserves_geometry():
bases = camera_bases()
distances = [room_reference(view_rays(256) @ basis)[0] for basis in bases]
merged, mask = merge_distances(distances, np.ones((12, 256, 256), bool), bases, 128, 64)
expected, _ = room_reference(panorama_directions(128, 64))
assert mask.all()
assert np.mean(np.abs(merged / expected - 1)) < 0.005
def test_input_alpha_survives_panorama_export(tmp_path, monkeypatch):
from PIL import Image
from prototype import main
rgba = np.full((16, 32, 4), 255, dtype=np.uint8)
rgba[:, :16, 3] = 0
source = tmp_path / "source.png"
destination = tmp_path / "result"
Image.fromarray(rgba).save(source)
monkeypatch.setattr("sys.argv", ["prototype", "--image", str(source), "--output", str(destination),
"--reference-only", "--size", "32", "--width", "32"])
main()
with Image.open(destination / "panorama.png") as exported:
np.testing.assert_array_equal(np.asarray(exported)[..., 3], rgba[..., 3])
with np.load(destination / "prediction.npz") as data:
assert np.all(data["alpha"][:, :16] == 0)
assert np.any(data["alpha"][:, 16:] > 0)
def test_depth_alpha_combines_mask_and_source_alpha_once():
from prototype import depth_alpha
image = np.ones((2, 4, 4), dtype=np.float32)
image[..., 3] = (0, 0.5, 0.75, 1)
mask = np.full((2, 4), 0.5, dtype=np.float32)
np.testing.assert_allclose(depth_alpha(image, mask), np.tile((0, 0.25, 0.375, 0.5), (2, 1)))

147
experiments/panorama_surface/validation.py Normal file
View File

@@ -0,0 +1,147 @@
"""Verify saved panorama assets using synthetic validity and depth inputs."""
import json
import bpy
import numpy as np
def socket_id(group, name):
return next(item.identifier for item in group.interface.items_tree
if item.item_type == "SOCKET" and item.in_out == "INPUT" and item.name == name)
def evaluated_mesh(obj):
obj.update_tag()
bpy.context.view_layer.update()
evaluated = obj.evaluated_get(bpy.context.evaluated_depsgraph_get())
return evaluated, evaluated.to_mesh()
def evaluated_points(obj):
evaluated, mesh = evaluated_mesh(obj)
points = np.empty(len(mesh.vertices) * 3, dtype=np.float32)
mesh.vertices.foreach_get("co", points)
counts = (len(mesh.vertices), len(mesh.edges), len(mesh.polygons))
evaluated.to_mesh_clear()
return points.reshape(-1, 3), counts
def test_image(name, alpha=1):
image = bpy.data.images.new(name, width=16, height=8, alpha=True, float_buffer=True)
image.colorspace_settings.name = "Non-Color"
image.alpha_mode = "CHANNEL_PACKED"
pixels = np.full((8, 16, 4), 3, dtype=np.float32)
pixels[..., 3] = alpha
image.pixels.foreach_set(pixels.ravel())
image.update()
return image
def verify(output):
full, empty, partial = test_image("Full"), test_image("Empty", 0), test_image("Combined Alpha", 0.25)
half = test_image("Half")
pixels = np.full((8, 16, 4), 3, dtype=np.float32)
pixels[..., 3] = 1
pixels[:, :8, 3] = 0
half.pixels.foreach_set(pixels.ravel())
half.update()
results = {}
try:
for obj in list(bpy.data.objects):
if obj.type != "MESH" or not obj.modifiers:
continue
modifier = obj.modifiers[0]
group = modifier.node_group
image_inputs = [item.name for item in group.interface.items_tree if item.item_type == "SOCKET"
and item.in_out == "INPUT" and item.socket_type == "NodeSocketImage"]
assert image_inputs == ["Distance"]
names = ("Subdivide", "Depth Scale", "Validity Threshold", "Distance")
keys = {name: socket_id(group, name) for name in names}
original = {name: modifier[key] for name, key in keys.items()}
def set_inputs(**values):
for name, value in values.items():
modifier[keys[name.replace("_", " ")]] = value
try:
set_inputs(Distance=full, Depth_Scale=0, Validity_Threshold=0.9)
face_counts = []
for level in (0, 2, 4):
set_inputs(Subdivide=level)
evaluated, mesh = evaluated_mesh(obj)
try:
assert len(mesh.polygons) == 6 * 4 ** level
assert all(len(face.vertices) == 4 for face in mesh.polygons)
assert len(mesh.vertices) - len(mesh.edges) + len(mesh.polygons) == 2
use = np.zeros(len(mesh.edges), dtype=int)
for loop in mesh.loops:
use[loop.edge_index] += 1
assert np.all(use == 2)
points = np.array([vertex.co[:] for vertex in mesh.vertices])
np.testing.assert_allclose(np.linalg.norm(points, axis=1), 1, atol=1e-5)
if level >= 2:
uv = mesh.attributes["UVMap"].data
assert all(np.ptp([uv[i].vector.x for i in face.loop_indices]) < 0.51
for face in mesh.polygons), "UV seam crosses the full image"
face_counts.append(len(mesh.polygons))
finally:
evaluated.to_mesh_clear()
sphere, _ = evaluated_points(obj)
set_inputs(Depth_Scale=1)
surface, _ = evaluated_points(obj)
np.testing.assert_allclose(surface, sphere * 3, atol=1e-5)
set_inputs(Depth_Scale=0.5)
middle, _ = evaluated_points(obj)
np.testing.assert_allclose(middle, sphere * 2, atol=1e-5)
set_inputs(Distance=empty)
assert evaluated_points(obj)[1][2] == 0
set_inputs(Distance=half)
retained = evaluated_points(obj)[1][2]
assert 0 < retained < face_counts[-1]
set_inputs(Distance=partial, Validity_Threshold=0.3)
assert evaluated_points(obj)[1][2] == 0, "Mask times alpha must be below threshold"
set_inputs(Validity_Threshold=0.2)
assert evaluated_points(obj)[1][2] == face_counts[-1]
finally:
for name, value in original.items():
modifier[keys[name]] = value
obj.update_tag()
points, counts = evaluated_points(obj)
assert np.isfinite(points).all()
assert counts[2] > 0
evaluated, mesh = evaluated_mesh(obj)
try:
directions = np.empty((counts[0], 3), dtype=np.float32)
uv = np.empty_like(directions)
mesh.attributes["panorama_direction"].data.foreach_get("vector", directions.ravel())
mesh.attributes["panorama_sample_uv"].data.foreach_get("vector", uv.ravel())
expected_u = np.arctan2(directions[:, 1], directions[:, 0]) / (2 * np.pi) % 1
delta_u = np.abs(uv[:, 0] - expected_u)
assert np.max(np.minimum(delta_u, 1 - delta_u)) < 1e-6
np.testing.assert_allclose(uv[:, 1], 1 - np.arccos(np.clip(directions[:, 2], -1, 1)) / np.pi, atol=1e-6)
image = original["Distance"]
width, height = image.size
pixels = np.array(image.pixels[:]).reshape(height, width, 4)
data_path = output / ("reference.npz" if obj.name == "Reference Room" else "prediction.npz")
with np.load(data_path) as data:
np.testing.assert_allclose(pixels[::-1, :, 3], data["alpha"], atol=0.0005)
x = np.clip((uv[:, 0] * width).astype(int), 0, width - 1)
y = np.clip((uv[:, 1] * height).astype(int), 0, height - 1)
radius = 1 + original["Depth Scale"] * (pixels[y, x, 0] - 1)
np.testing.assert_allclose(points, directions * radius[:, None], rtol=2e-6, atol=2e-5)
finally:
evaluated.to_mesh_clear()
results[obj.name] = {"subdivide_levels": [0, 2, 4], "quad_face_counts": face_counts,
"closed_quad_sphere": True, "uv_seam_verified": True,
"depth_scale_verified": True, "mask_alpha_verified": True,
"image_sampling_verified": True,
"single_depth_image_verified": True,
"packed_depth_alpha_verified": True,
"vertices": counts[0], "faces": counts[2]}
assert results
finally:
for image in (full, empty, partial, half):
bpy.data.images.remove(image)
(output / "blender_validation.json").write_text(json.dumps(results, indent=2), encoding="utf-8")
print(json.dumps(results, indent=2), flush=True)