feat: Add surface generation and media handling
- Implemented surface generation functionality in `surface.py`, including geometry estimation and texture writing. - Introduced media input handling in `input.py`, supporting image and video frame extraction. - Added model management for geometry adapters in `models/__init__.py`. - Created BEN2 and DA3 model inference logic in `ben2.py` and `da3.py`, respectively. - Developed ONNX model inference utilities for BEN2 and DA3 in `onnx_ben2.py` and `onnx_da3.py`. - Added support for MoGe-2 model inference in `moge2.py` with necessary ONNX utilities. - Implemented output handling for depth and surface data in `depth.py` and `surface.py`. - Created debug output utilities for DA3 and MoGe-2 predictions in `debug_da3.py` and `debug_moge2.py`. - Enhanced ONNX runtime session management in `onnx_runtime.py`. - Updated `__init__.py` files to include new modules and functions for better organization.
This commit is contained in:
75
src/snapform/server/jobs/surface.py
Normal file
75
src/snapform/server/jobs/surface.py
Normal file
@@ -0,0 +1,75 @@
|
||||
from ..geometry import foreground_mesh_mask
|
||||
from ..models import geometry_adapter, moge2
|
||||
from ..geometry import build_surface
|
||||
from ..outputs.surface import (
|
||||
write_normal_texture,
|
||||
write_surface_archive,
|
||||
write_surface_texture,
|
||||
)
|
||||
from .common import input_frames
|
||||
from .foreground import write_optional
|
||||
|
||||
|
||||
def run(context, parameters):
|
||||
context.progress(0.02, "Preparing surface input")
|
||||
context.directory.mkdir(parents=True, exist_ok=True)
|
||||
model_manager = context.resource("model_manager")
|
||||
adapter = geometry_adapter(parameters["model"])
|
||||
with input_frames(context, parameters) as images:
|
||||
if len(images) != 1:
|
||||
raise ValueError("Convert to Surface currently supports one image")
|
||||
context.progress(0.12, "Estimating surface geometry")
|
||||
if adapter is moge2:
|
||||
prediction = adapter.infer(
|
||||
model_manager,
|
||||
parameters,
|
||||
images,
|
||||
include_points=True,
|
||||
progress_callback=lambda index, total: context.progress(
|
||||
0.12 + 0.58 * index / total,
|
||||
f"Estimating MoGe-2 geometry ({index + 1}/{total})",
|
||||
),
|
||||
cancel_check=context.check_cancelled,
|
||||
)
|
||||
else:
|
||||
prediction = adapter.infer(
|
||||
model_manager,
|
||||
parameters,
|
||||
images,
|
||||
progress_callback=lambda index, total: context.progress(
|
||||
0.12 + 0.58 * index / total,
|
||||
f"Estimating DA3 geometry ({index + 1}/{total})",
|
||||
),
|
||||
cancel_check=context.check_cancelled,
|
||||
)
|
||||
context.check_cancelled()
|
||||
frame = adapter.surface_frame(prediction)
|
||||
foreground_paths = write_optional(
|
||||
context,
|
||||
parameters,
|
||||
images,
|
||||
progress_start=0.72,
|
||||
progress_span=0.08,
|
||||
)
|
||||
foreground_mask = None
|
||||
if foreground_paths:
|
||||
foreground_mask = foreground_mesh_mask(
|
||||
foreground_paths[0],
|
||||
frame.depth.shape,
|
||||
)
|
||||
context.progress(0.82, "Building surface")
|
||||
mesh = build_surface(frame, foreground_mask=foreground_mask)
|
||||
context.progress(0.92, "Writing surface")
|
||||
write_surface_archive(mesh, context.directory / "mesh.npz")
|
||||
write_surface_texture(
|
||||
foreground_paths[0] if foreground_paths else images[0],
|
||||
context.directory / "texture.png",
|
||||
)
|
||||
if adapter is moge2:
|
||||
write_normal_texture(
|
||||
prediction.normal[0],
|
||||
context.directory / "normal.png",
|
||||
)
|
||||
context.check_cancelled()
|
||||
context.progress(1.0, "Surface is ready")
|
||||
return {"surface": "mesh.npz"}
|
||||
Reference in New Issue
Block a user