feat: Add surface generation and media handling

- Implemented surface generation functionality in `surface.py`, including geometry estimation and texture writing.
- Introduced media input handling in `input.py`, supporting image and video frame extraction.
- Added model management for geometry adapters in `models/__init__.py`.
- Created BEN2 and DA3 model inference logic in `ben2.py` and `da3.py`, respectively.
- Developed ONNX model inference utilities for BEN2 and DA3 in `onnx_ben2.py` and `onnx_da3.py`.
- Added support for MoGe-2 model inference in `moge2.py` with necessary ONNX utilities.
- Implemented output handling for depth and surface data in `depth.py` and `surface.py`.
- Created debug output utilities for DA3 and MoGe-2 predictions in `debug_da3.py` and `debug_moge2.py`.
- Enhanced ONNX runtime session management in `onnx_runtime.py`.
- Updated `__init__.py` files to include new modules and functions for better organization.
This commit is contained in:
2026-08-18 23:27:10 +08:00
parent 9dfa49602c
commit 835225d264
62 changed files with 2126 additions and 2318 deletions

View File

@@ -0,0 +1,75 @@
from ..geometry import foreground_mesh_mask
from ..models import geometry_adapter, moge2
from ..geometry import build_surface
from ..outputs.surface import (
write_normal_texture,
write_surface_archive,
write_surface_texture,
)
from .common import input_frames
from .foreground import write_optional
def run(context, parameters):
context.progress(0.02, "Preparing surface input")
context.directory.mkdir(parents=True, exist_ok=True)
model_manager = context.resource("model_manager")
adapter = geometry_adapter(parameters["model"])
with input_frames(context, parameters) as images:
if len(images) != 1:
raise ValueError("Convert to Surface currently supports one image")
context.progress(0.12, "Estimating surface geometry")
if adapter is moge2:
prediction = adapter.infer(
model_manager,
parameters,
images,
include_points=True,
progress_callback=lambda index, total: context.progress(
0.12 + 0.58 * index / total,
f"Estimating MoGe-2 geometry ({index + 1}/{total})",
),
cancel_check=context.check_cancelled,
)
else:
prediction = adapter.infer(
model_manager,
parameters,
images,
progress_callback=lambda index, total: context.progress(
0.12 + 0.58 * index / total,
f"Estimating DA3 geometry ({index + 1}/{total})",
),
cancel_check=context.check_cancelled,
)
context.check_cancelled()
frame = adapter.surface_frame(prediction)
foreground_paths = write_optional(
context,
parameters,
images,
progress_start=0.72,
progress_span=0.08,
)
foreground_mask = None
if foreground_paths:
foreground_mask = foreground_mesh_mask(
foreground_paths[0],
frame.depth.shape,
)
context.progress(0.82, "Building surface")
mesh = build_surface(frame, foreground_mask=foreground_mask)
context.progress(0.92, "Writing surface")
write_surface_archive(mesh, context.directory / "mesh.npz")
write_surface_texture(
foreground_paths[0] if foreground_paths else images[0],
context.directory / "texture.png",
)
if adapter is moge2:
write_normal_texture(
prediction.normal[0],
context.directory / "normal.png",
)
context.check_cancelled()
context.progress(1.0, "Surface is ready")
return {"surface": "mesh.npz"}