icrdr/AnyImage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(node_assets): add image plane geometry nodes and validation scripts

Browse Source 
- Implemented `image_plane.py` to build the Image Plane Geometry Nodes group with customizable parameters for geometry, subdivide, and thickness.
- Created `image_plane_validation.py` to validate the functionality of the image plane, ensuring correct geometry generation and material assignment.
- Developed `material_layers.py` to construct shader node groups for image materials, including depth and normal handling.
- Authored `plane_layouts.py` to define node arrangements for image geometry assets.
- Introduced `shadeless.py` to create a shadeless shader node group that displays color without scene lighting while preserving shadows and reflections.
This commit is contained in:
马南
2026-09-08 23:24:27 +08:00
parent 196b964efc
commit 4c902ed7db
155 changed files with 9700 additions and 10393 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/anyimage/__init__.py
View File

@@ -10,14 +10,7 @@ from .operators.image_edit_tool import (
MaskTool,
RectifyTool,
)
from .panel import (
Ben2RemovalDebugPanel,
Da3OutputDebugPanel,
DebugPanel,
Moge2ApiDebugPanel,
ServerPanel,
UpscaleDebugPanel,
)
from .panel import ServerPanel
from .preferences import AnyImagePreferences
from .properties import AnyImageSettings
from .runtime import runtime
@@ -30,11 +23,6 @@ CLASSES = (
*OPERATOR_CLASSES,
AnyImageImageMenu,
ServerPanel,
DebugPanel,
Moge2ApiDebugPanel,
Da3OutputDebugPanel,
Ben2RemovalDebugPanel,
UpscaleDebugPanel,
)

BIN
src/anyimage/assets/O_AnyImage.blend LFS
View File

Binary file not shown.

13
src/anyimage/common/ai.py
View File

@@ -79,19 +79,6 @@ def require_model(model_key):
return model
def da3_parameters(settings, path, model, process_res, start_frame):
return {
"input": str(path),
"model": model,
"process_res": process_res,
"process_res_method": "upper_bound_resize",
"video_frames": settings.video_frames,
"start_frame": start_frame,
"device": production_device().lower(),
"ref_view_strategy": settings.ref_view_strategy,
}
def moge2_parameters(settings, path, model, resolution_level, start_frame):
return {
"input": str(path),

2
src/anyimage/common/material.py
View File

@@ -281,5 +281,3 @@ def set_displacement_only(material):
if owner is not None and hasattr(owner, property_name):
setattr(owner, property_name, "DISPLACEMENT")
return

29
src/anyimage/licenses/scikit-image.txt
View File

@@ -1,29 +0,0 @@
approximate_polygon from scikit-image 0.25.2, skimage/measure/_polygon.py
Copyright: 2009-2022 the scikit-image team
License: BSD-3-Clause
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the University nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE HOLDERS OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

2
src/anyimage/menu.py
View File

@@ -16,7 +16,7 @@ class AnyImageImageMenu(bpy.types.Menu):
layout.operator(
"anyimage.convert_to_plane",
text="Convert to Plane",
icon="MESH_PLANE",
icon="IMAGE_PLANE",
)
layout.operator(
"anyimage.convert_to_depth_plane",

5
src/anyimage/operators/__init__.py
View File

@@ -3,7 +3,6 @@ from .cutout_tool import (
CutoutSelectionToShape,
SelectCutoutSelection,
)
from .debug import DebugBen2, DebugDa3, DebugMoge2, DebugUpscale
from .image_edit_tool import CLASSES as IMAGE_EDIT_TOOL_CLASSES
from .convert_to_plane import (
ConvertToDepthPlane,
@@ -25,10 +24,6 @@ CLASSES = (
OpenAIEnvironmentSettings,
DownloadModel,
DownloadRequiredModels,
DebugDa3,
DebugMoge2,
DebugBen2,
DebugUpscale,
GenerateDepthPlane,
CutoutSelectionToShape,
SelectCutoutSelection,

4
src/anyimage/operators/clipboard_image/actions.py
View File

@@ -147,11 +147,11 @@ def add_image_plane(
if width <= 0 or height <= 0:
raise PasteTargetError("The clipboard image has invalid dimensions")
from ..convert_to_plane.mesh_plane import create_mesh_plane_object
from ..convert_to_plane.image_plane import create_image_plane_object
half_width, half_height = _plane_half_dimensions(width, height)
material = _create_image_material(image, shadeless, scene=context.scene)
plane = create_mesh_plane_object(
plane = create_image_plane_object(
context,
image.name,
(-half_width, half_width, -half_height, half_height),

2
src/anyimage/operators/convert_to_plane/__init__.py
View File

@@ -1,4 +1,4 @@
"""Mesh Plane and Depth Plane conversion operators."""
"""Rectangular Plane and Depth Plane conversion operators."""
from .conversion import (
ConvertToDepthPlane,

6
src/anyimage/operators/convert_to_plane/conversion.py
View File

@@ -46,7 +46,7 @@ class GenerateDepthPlane(JobOperatorBase, bpy.types.Operator):
bl_idname = "anyimage.generate_depth_plane"
bl_label = "Generate Depth Plane"
job_type = "generate-depth-plane-geometry"
starting_message = "Preparing Depth Plane"
starting_message = "Preparing depth conversion"
input_path: bpy.props.StringProperty(options=_HIDDEN)
source_object_name: bpy.props.StringProperty(options=_HIDDEN)
@@ -94,7 +94,7 @@ class GenerateDepthPlane(JobOperatorBase, bpy.types.Operator):
def start_depth_plane_job(operator, context):
source_object = context.object
if video_total_frames(source_object.data) != 1:
operator.report({"ERROR"}, "Depth Plane currently supports still images")
operator.report({"ERROR"}, "Depth conversion currently supports still images")
return {"CANCELLED"}
setup_result = invoke_ai_setup_if_needed()
if setup_result is not None:
@@ -157,7 +157,7 @@ class ConvertToPlane(bpy.types.Operator):
class ConvertToDepthPlane(bpy.types.Operator):
bl_idname = "anyimage.convert_to_depth_plane"
bl_label = "Convert to Depth Plane"
bl_description = "Convert the complete Image Empty into a front-projected relief"
bl_description = "Convert the complete Image Empty into a Relief or Camera depth plane"
bl_options = {"REGISTER", "UNDO"}
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=6, min=0, max=10)

22
src/anyimage/operators/convert_to_plane/mesh_plane.py → src/anyimage/operators/convert_to_plane/image_plane.py
View File

@@ -1,4 +1,4 @@
"""Create Mesh Plane objects used by conversion and clipboard paste."""
"""Create Image Plane objects used by conversion and clipboard paste."""
import bpy
@@ -6,10 +6,10 @@ from ...common.object import modifier_input_identifier, set_modifier_input
from ...common.node import load_node_group
MESH_PLANE_NODE_GROUP_NAME = "O Mesh Plane"
IMAGE_PLANE_NODE_GROUP_NAME = "O Image Plane"
def build_mesh_plane_matrix(matrix_world, bounds):
def build_image_plane_matrix(matrix_world, bounds):
from mathutils import Matrix
x_min, x_max, y_min, y_max = bounds
@@ -17,7 +17,7 @@ def build_mesh_plane_matrix(matrix_world, bounds):
return matrix_world @ Matrix.Translation(center)
def create_mesh_plane_mesh(name, bounds):
def create_image_plane_mesh(name, bounds):
x_min, x_max, y_min, y_max = bounds
if x_max <= x_min or y_max <= y_min:
raise ValueError("The plane has invalid dimensions")
@@ -46,14 +46,14 @@ def create_mesh_plane_mesh(name, bounds):
return mesh
def add_mesh_plane_processing(
def add_image_plane_processing(
plane_object,
subdivisions=0,
thickness=0.0,
):
node_group = load_node_group(MESH_PLANE_NODE_GROUP_NAME)
node_group = load_node_group(IMAGE_PLANE_NODE_GROUP_NAME)
modifier = plane_object.modifiers.new(
name=MESH_PLANE_NODE_GROUP_NAME,
name=IMAGE_PLANE_NODE_GROUP_NAME,
type="NODES",
)
modifier.node_group = node_group
@@ -68,7 +68,7 @@ def add_mesh_plane_processing(
return modifier
def create_mesh_plane_object(
def create_image_plane_object(
context,
name,
bounds,
@@ -77,7 +77,7 @@ def create_mesh_plane_object(
subdivisions=0,
thickness=0.0,
):
mesh = create_mesh_plane_mesh(name, bounds)
mesh = create_image_plane_mesh(name, bounds)
mesh.materials.append(material)
plane = bpy.data.objects.new(name, mesh)
context.collection.objects.link(plane)
@@ -85,8 +85,8 @@ def create_mesh_plane_object(
from mathutils import Matrix
matrix_world = Matrix.Identity(4)
plane.matrix_world = build_mesh_plane_matrix(matrix_world, bounds)
add_mesh_plane_processing(
plane.matrix_world = build_image_plane_matrix(matrix_world, bounds)
add_image_plane_processing(
plane,
subdivisions=subdivisions,
thickness=thickness,

81
src/anyimage/operators/convert_to_plane/object.py
View File

@@ -23,16 +23,13 @@ from ...common.object import (
set_modifier_input,
)
from ...common.node import load_node_group
from .mesh_plane import (
build_mesh_plane_matrix,
create_mesh_plane_mesh,
create_mesh_plane_object,
from .image_plane import (
build_image_plane_matrix,
create_image_plane_mesh,
create_image_plane_object,
)
DEPTH_PLANE_NODE_GROUP_NAME = "O Image Depth Plane"
def create_color_image(source_object):
source = source_object.data
image = source.copy()
@@ -49,7 +46,7 @@ def create_color_image(source_object):
def create_plane_object(context, source_object, subdivisions, material):
source_name = source_object.name
plane = create_mesh_plane_object(
plane = create_image_plane_object(
context,
source_name,
image_empty_bounds(source_object),
@@ -71,35 +68,42 @@ def create_depth_plane_object(
depth_image,
depth_metadata,
):
node_group = load_node_group("O Image Depth Plane")
source_name = source_object.name
bounds = image_empty_bounds(source_object)
mesh = create_mesh_plane_mesh(source_name, bounds)
for polygon in mesh.polygons:
polygon.use_smooth = True
mesh.materials.append(material)
plane = bpy.data.objects.new(source_name, mesh)
context.collection.objects.link(plane)
plane.matrix_world = build_mesh_plane_matrix(source_object.matrix_world, bounds)
plane["anyimage_mesh_shape"] = "DEPTH_PLANE"
mesh = create_image_plane_mesh(source_name, bounds)
plane = None
try:
for polygon in mesh.polygons:
polygon.use_smooth = True
mesh.materials.append(material)
plane = bpy.data.objects.new(source_name, mesh)
context.collection.objects.link(plane)
plane.matrix_world = build_image_plane_matrix(source_object.matrix_world, bounds)
plane["anyimage_mesh_shape"] = "DEPTH_PLANE"
node_group = load_node_group(DEPTH_PLANE_NODE_GROUP_NAME)
modifier = plane.modifiers.new(name=node_group.name, type="NODES")
modifier.node_group = node_group
modifier.show_group_selector = False
uniform_scale = depth_uniform_scale(source_object, depth_metadata)
for name, value in (
("Subdivide", max(0, min(10, int(subdivisions)))),
("Thickness", 0.1),
("Depth Scale", 1.0),
("Reference Depth", depth_metadata["reference_depth"] * uniform_scale),
("Uniform Scale", uniform_scale),
("Depth Image", depth_image),
):
set_modifier_input(
modifier,
modifier_input_identifier(node_group, name),
value,
)
modifier = plane.modifiers.new(name=node_group.name, type="NODES")
modifier.node_group = node_group
modifier.show_group_selector = False
uniform_scale = depth_uniform_scale(source_object, depth_metadata)
for name, value in (
("Subdivide", max(0, min(10, int(subdivisions)))),
("Depth Scale", 1.0),
("Reference Depth", depth_metadata["reference_depth"] * uniform_scale),
("Uniform Scale", uniform_scale),
("Depth Image", depth_image),
):
set_modifier_input(
modifier,
modifier_input_identifier(node_group, name),
value,
)
except Exception:
if plane is not None:
bpy.data.objects.remove(plane, do_unlink=True)
if mesh.users == 0:
bpy.data.meshes.remove(mesh)
raise
return finalize_object_result(context, source_object, plane)
@@ -113,7 +117,7 @@ def create_depth_plane_from_result(context, result, operator):
try:
depth_metadata = load_depth_metadata(result.file("depth_metadata"))
depth_image = load_depth_result_image(
result.file("z_depth"),
result.file("depth"),
source_object,
depth_metadata,
)
@@ -141,9 +145,10 @@ def create_depth_plane_from_result(context, result, operator):
except Exception:
if material is not None and material.users == 0:
bpy.data.materials.remove(material, do_unlink=True)
for image in (depth_image, normal_image):
for image in (color_image, depth_image, normal_image):
if image is not None and image.users == 0:
bpy.data.images.remove(image, do_unlink=True)
raise
if "FINISHED" not in bpy.ops.ed.undo_push(message="Convert to Depth Plane"):
raise RuntimeError("Unable to create the Convert to Depth Plane Undo step")
label = "Convert to Depth Plane"
if "FINISHED" not in bpy.ops.ed.undo_push(message=label):
raise RuntimeError(f"Unable to create the {label} Undo step")

57
src/anyimage/operators/cutout_tool/main.py
View File

@@ -46,8 +46,6 @@ from ...common.viewport import (
ImageGesture,
image_edit_drag_keymap,
report_image_edit_exception,
resolve_image_edit_click,
serialize_matrix,
)
from ...common.selection import (
ImageEditWarning,
@@ -82,12 +80,8 @@ def normal_mode_for_shape(shape, generate_normal):
return NormalMode.TANGENT
def depth_mode_for_shape(shape):
if shape == "DEPTH_SOLID":
return "VECTOR"
if shape == "DEPTH_BALLOON":
return "Z"
return "NONE"
def shape_uses_depth(shape):
return shape in {"DEPTH_SOLID", "DEPTH_BALLOON"}
def create_cutout_shape(
@@ -322,19 +316,19 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
def request(self, context):
require_environment()
source_path = require_input_path(self.input_path)
depth_mode = depth_mode_for_shape(self.shape)
generate_depth = shape_uses_depth(self.shape)
normal_mode = normal_mode_for_shape(self.shape, self.generate_normal)
if depth_mode != "NONE" or normal_mode != NormalMode.NONE:
if generate_depth or normal_mode != NormalMode.NONE:
require_model(self.model)
parameters = {
"input": str(source_path),
"device": production_device().lower(),
"depth_mode": depth_mode,
"generate_depth": generate_depth,
"normal_mode": normal_mode.value,
"max_input_size": int(self.max_ai_input_size),
"alpha_threshold": float(self.alpha_threshold),
}
if depth_mode != "NONE" or normal_mode != NormalMode.NONE:
if generate_depth or normal_mode != NormalMode.NONE:
start_frame = source_start_frame(self.source_object_name)
parameters.update(
model=self.model,
@@ -388,11 +382,10 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
if normal_mode == NormalMode.OBJECT:
orient_object_normal_image(normal_image, inward_axis)
depth_metadata = None
depth_mode = depth_mode_for_shape(self.shape)
if depth_mode != "NONE":
if shape_uses_depth(self.shape):
depth_metadata = load_depth_metadata(result.file("depth_metadata"))
depth_image = load_depth_result_image(
result.file(f"{depth_mode.lower()}_depth"),
result.file("depth"),
source_object,
depth_metadata,
)
@@ -437,7 +430,6 @@ class SelectCutoutSelection(ImageGesture, bpy.types.Operator):
source_object_name: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
source_matrix_data: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
full_image: bpy.props.BoolProperty(default=False, options={"HIDDEN", "SKIP_SAVE"})
_pie_event = None
@@ -448,25 +440,7 @@ class SelectCutoutSelection(ImageGesture, bpy.types.Operator):
def invoke(self, context, event):
self._pie_event = event
if not self.full_image and event.value != "DOUBLE_CLICK":
return super().invoke(context, event)
source_object, should_edit = resolve_image_edit_click(
context, event, self.report
)
if not should_edit:
return {"FINISHED"} if source_object is not None else {"CANCELLED"}
if source_object is None:
return {"CANCELLED"}
if video_total_frames(source_object.data) > 1:
self.report({"WARNING"}, "Cutout only supports a still image")
return {"CANCELLED"}
self.source_object_name = source_object.name
self.source_matrix_data = serialize_matrix(source_object.matrix_world)
width, height = (int(value) for value in source_object.data.size)
selection_path = SelectionPath(
points=((0, 0), (width, 0), (width, height), (0, height)),
)
return self._submit_selection_path(context, selection_path)
return super().invoke(context, event)
def modal(self, context, event):
self._pie_event = event
@@ -495,16 +469,9 @@ class CutoutTool(bpy.types.WorkSpaceTool):
bl_description = "Lasso an Image Empty and choose a mesh shape"
bl_icon = "ops.mesh.primitive_sphere_add_gizmo"
bl_operator = SelectCutoutSelection.bl_idname
bl_keymap = (
(
SelectCutoutSelection.bl_idname,
{"type": "LEFTMOUSE", "value": "DOUBLE_CLICK"},
{"properties": [("full_image", True)]},
),
*image_edit_drag_keymap(
SelectCutoutSelection.bl_idname,
deselect_on_empty=False,
),
bl_keymap = image_edit_drag_keymap(
SelectCutoutSelection.bl_idname,
deselect_on_empty=False,
)
def draw_settings(context, layout, _tool):

129
src/anyimage/operators/debug.py
View File

@@ -1,129 +0,0 @@
from pathlib import Path
import bpy
from ..runtime import JobOperatorBase
from ..properties import BEN2_MODEL_KEY, REALESRGAN_GENERAL_WDN_MODEL_KEY
from ..common.ai import (
require_background_input_path,
da3_parameters,
require_input_path,
moge2_parameters,
require_environment,
require_model,
)
def _open_job_directory(directory):
directory = Path(directory)
if not directory.is_dir():
raise RuntimeError(f"Job directory does not exist: {directory}")
result = bpy.ops.wm.path_open(filepath=str(directory))
if "FINISHED" not in result:
raise RuntimeError(f"Unable to open job directory: {directory}")
class DebugDa3(JobOperatorBase, bpy.types.Operator):
bl_idname = "anyimage.debug_da3"
bl_label = "Debug DA3"
job_type = "debug-da3"
starting_message = "Starting DA3 API inference"
def request(self, context):
require_environment()
settings = context.scene.anyimage_settings
model = settings.model
require_model(model)
return {
**da3_parameters(
settings,
require_input_path(settings.input_path),
model,
settings.process_res,
1,
),
"device": settings.device.lower(),
"inference_mode": "joint",
"remove_background_model": "",
}
def response(self, _context, result):
_open_job_directory(result.directory)
return f"Saved DA3 API prediction to {result.directory}"
class DebugMoge2(JobOperatorBase, bpy.types.Operator):
bl_idname = "anyimage.debug_moge2"
bl_label = "Debug MoGe-2"
job_type = "debug-moge2"
starting_message = "Starting MoGe-2 inference"
def request(self, context):
require_environment()
settings = context.scene.anyimage_settings
require_model(settings.moge_model)
return {
**moge2_parameters(
settings,
require_input_path(settings.input_path),
settings.moge_model,
settings.moge_resolution_level,
1,
),
"device": settings.device.lower(),
}
def response(self, _context, result):
_open_job_directory(result.directory)
return f"Exported MoGe-2 data to {result.directory}"
class DebugBen2(JobOperatorBase, bpy.types.Operator):
bl_idname = "anyimage.debug_ben2"
bl_label = "Debug BEN2"
job_type = "debug-ben2"
starting_message = "Starting BEN2 background removal"
def request(self, context):
require_environment()
settings = context.scene.anyimage_settings
require_model(BEN2_MODEL_KEY)
return {
"input": str(require_background_input_path(settings.background_input_path)),
"model": BEN2_MODEL_KEY,
"device": settings.device.lower(),
"video_frames": settings.video_frames,
"start_frame": 1,
}
def response(self, _context, result):
_open_job_directory(result.directory)
return f"Removed background to {result.directory}"
class DebugUpscale(JobOperatorBase, bpy.types.Operator):
bl_idname = "anyimage.debug_upscale"
bl_label = "Debug Upscale"
job_type = "debug-upscale"
starting_message = "Starting Upscale inference"
model: bpy.props.StringProperty(
default=REALESRGAN_GENERAL_WDN_MODEL_KEY,
options={"HIDDEN", "SKIP_SAVE"},
)
def request(self, context):
require_environment()
settings = context.scene.anyimage_settings
require_model(self.model)
return {
"input": str(require_input_path(settings.input_path)),
"model": self.model,
"device": settings.device.lower(),
"video_frames": settings.video_frames,
"start_frame": 1,
}
def response(self, _context, result):
_open_job_directory(result.directory)
return f"Saved upscale API prediction to {result.directory}"

156
src/anyimage/panel.py
View File

@@ -1,11 +1,6 @@
import bpy
from .runtime import runtime
from .properties import (
BEN2_MODEL_KEY,
model_ready,
)
from .server.model_catalog import UPSCALE_MODELS
class ServerPanel(bpy.types.Panel):
bl_label = "Job Server"
@@ -110,154 +105,3 @@ class ServerPanel(bpy.types.Panel):
text="",
icon="TRASH",
)
class DebugPanel(bpy.types.Panel):
bl_label = "Debug"
bl_idname = "ANYIMAGE_PT_debug"
bl_space_type = "VIEW_3D"
bl_region_type = "UI"
bl_category = "AnyImage"
bl_options = {"DEFAULT_CLOSED"}
bl_order = 2
def draw(self, context):
layout = self.layout
settings = context.scene.anyimage_settings
layout.prop(settings, "device")
def draw_da3_input_settings(layout, settings, model_property):
layout.prop(settings, model_property)
layout.prop(settings, "input_path")
input_path = bpy.path.abspath(settings.input_path)
if input_path.lower().endswith(
(
".mp4",
".mov",
".avi",
".mkv",
".webm",
".m4v",
".mts",
".mpg",
".mpeg",
)
):
layout.prop(settings, "video_frames")
layout.prop(settings, "process_res")
layout.prop(settings, "process_res_method")
layout.prop(settings, "ref_view_strategy")
class Da3OutputDebugPanel(bpy.types.Panel):
bl_label = "DA3"
bl_idname = "ANYIMAGE_PT_debug_da3_output"
bl_parent_id = DebugPanel.bl_idname
bl_space_type = "VIEW_3D"
bl_region_type = "UI"
bl_category = "AnyImage"
bl_options = {"DEFAULT_CLOSED"}
bl_order = 1
def draw(self, context):
layout = self.layout
settings = context.scene.anyimage_settings
draw_da3_input_settings(layout, settings, "model")
export_row = layout.row()
export_row.enabled = (
not runtime.server_busy()
and model_ready(settings.model)
)
export_row.operator(
"anyimage.debug_da3",
text="Run DA3 API",
icon="EXPORT",
)
class Moge2ApiDebugPanel(bpy.types.Panel):
bl_label = "MoGe-2"
bl_idname = "ANYIMAGE_PT_debug_moge2_api"
bl_parent_id = DebugPanel.bl_idname
bl_space_type = "VIEW_3D"
bl_region_type = "UI"
bl_category = "AnyImage"
bl_options = {"DEFAULT_CLOSED"}
bl_order = 0
def draw(self, context):
layout = self.layout
settings = context.scene.anyimage_settings
layout.prop(settings, "input_path")
layout.prop(settings, "moge_model")
layout.prop(settings, "moge_resolution_level")
run_row = layout.row()
run_row.enabled = (
not runtime.server_busy()
and model_ready(settings.moge_model)
)
run_row.operator(
"anyimage.debug_moge2",
text="Run MoGe-2 API",
icon="EXPORT",
)
class Ben2RemovalDebugPanel(bpy.types.Panel):
bl_label = "BEN2"
bl_idname = "ANYIMAGE_PT_debug_ben2_removal"
bl_parent_id = DebugPanel.bl_idname
bl_space_type = "VIEW_3D"
bl_region_type = "UI"
bl_category = "AnyImage"
bl_options = {"DEFAULT_CLOSED"}
bl_order = 2
def draw(self, context):
layout = self.layout
settings = context.scene.anyimage_settings
layout.prop(settings, "background_input_path")
background_row = layout.row()
background_row.enabled = (
not runtime.server_busy()
and model_ready(BEN2_MODEL_KEY)
)
background_row.operator(
"anyimage.debug_ben2",
text="Remove Background",
icon="IMAGE_ALPHA",
)
class UpscaleDebugPanel(bpy.types.Panel):
bl_label = "Upscale"
bl_idname = "ANYIMAGE_PT_debug_upscale"
bl_parent_id = DebugPanel.bl_idname
bl_space_type = "VIEW_3D"
bl_region_type = "UI"
bl_category = "AnyImage"
bl_options = {"DEFAULT_CLOSED"}
bl_order = 3
def draw(self, context):
layout = self.layout
settings = context.scene.anyimage_settings
layout.prop(settings, "input_path")
input_path = bpy.path.abspath(settings.input_path)
if input_path.lower().endswith(
(".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".mts", ".mpg", ".mpeg")
):
layout.prop(settings, "video_frames")
for model in UPSCALE_MODELS.values():
row = layout.row()
row.enabled = not runtime.server_busy() and model_ready(model.key)
operator = row.operator(
"anyimage.debug_upscale",
text=f"Run {model.label} API",
icon="EXPORT",
)
operator.model = model.key

106
src/anyimage/properties.py
View File

@@ -11,7 +11,6 @@ BEN2_MODEL_KEY = "BEN2_BASE"
REALESRGAN_GENERAL_WDN_MODEL_KEY = shared_model_catalog.DEFAULT_UPSCALE_MODEL_KEY
SAFMN_MODEL_KEY = shared_model_catalog.SAFMN_MODEL.key
REALESRGAN_X4PLUS_MODEL_KEY = shared_model_catalog.REALESRGAN_X4PLUS.key
DEFAULT_DA3_MODEL_KEY = "BASE"
DEFAULT_MOGE2_MODEL_KEY = "MOGE2_VITS_NORMAL"
REQUIRED_MODEL_KEYS = (
DEFAULT_MOGE2_MODEL_KEY,
@@ -145,33 +144,8 @@ def _model_items(models):
]
def model_enum_items(_owner, _context):
return _model_items(
model
for model in model_catalog()
if model.get("family") == "da3" and model.get("enabled")
)
def moge2_model_enum_items(_owner, _context):
return _model_items(
model for model in model_catalog() if model.get("family") == "moge2"
)
def depth_plane_model_enum_items(_owner, _context):
models = [model for model in model_catalog() if model.get("family") == "moge2"]
models.sort(key=lambda model: model.get("family") != "moge2")
items = _model_items(models)
items.append(
("ORIGINAL", "Original", "Use the original image as depth", len(items))
)
return items
def geometry_model_enum_items(_owner, _context):
models = [model for model in model_catalog() if model.get("family") == "moge2"]
models.sort(key=lambda model: model.get("family") != "moge2")
return _model_items(models)
@@ -302,16 +276,6 @@ class AnyImageSettings(bpy.types.PropertyGroup):
mask_gesture: mask_gesture_property()
mask_mode: mask_mode_property()
mask_radius: mask_radius_property()
input_path: bpy.props.StringProperty(
name="Image or Video",
description="A single image or video file used for reconstruction",
subtype="FILE_PATH",
)
background_input_path: bpy.props.StringProperty(
name="Image",
description="A single image used to test BEN2 background removal",
subtype="FILE_PATH",
)
video_frames: bpy.props.IntProperty(
name="Extract Frames",
description="Maximum number of consecutive frames extracted from the start of a video or frame sequence",
@@ -319,73 +283,3 @@ class AnyImageSettings(bpy.types.PropertyGroup):
min=1,
soft_max=64,
)
model: bpy.props.EnumProperty(
name="Model",
description="Depth Anything 3 model",
items=model_enum_items,
)
device: bpy.props.EnumProperty(
name="Debug Device",
description=(
"Inference device used by Debug tasks only; production tasks use "
"the Device setting in the Add-on Preferences"
),
items=device_enum_items,
default=0,
)
process_res: bpy.props.IntProperty(
name="Process Res",
description="DA3 base processing resolution",
default=504,
min=1,
soft_max=2048,
)
process_res_method: bpy.props.EnumProperty(
name="Process Res Method",
description="DA3 image resize method",
items=(
(
"upper_bound_resize",
"Upper Bound Resize",
"Make Process Res the longer side",
),
(
"lower_bound_resize",
"Lower Bound Resize",
"Make Process Res the shorter side",
),
),
default="upper_bound_resize",
)
moge_resolution_level: bpy.props.IntProperty(
name="Resolution Level",
description="MoGe-2 inference resolution level; higher is finer and slower",
default=5,
min=0,
max=9,
)
moge_model: bpy.props.EnumProperty(
name="Model",
description="MoGe-2 model used for depth estimation",
items=moge2_model_enum_items,
default=0,
)
ref_view_strategy: bpy.props.EnumProperty(
name="Ref View Strategy",
description="DA3 reference view selection strategy",
items=(
("first", "First", "Always use the first view"),
("middle", "Middle", "Use the middle view"),
(
"saddle_balanced",
"Saddle Balanced",
"Select a view with balanced feature metrics",
),
(
"saddle_sim_range",
"Saddle Sim Range",
"Select the view with the largest similarity range",
),
),
default="saddle_balanced",
)

32
src/anyimage/server/app.py
View File

@@ -2,18 +2,6 @@
from blendjob import JobServer
from .jobs.debug_predictions import (
run_ben2 as _debug_ben2,
)
from .jobs.debug_predictions import (
run_da3 as _debug_da3,
)
from .jobs.debug_predictions import (
run_moge2 as _debug_moge2,
)
from .jobs.debug_predictions import (
run_upscale as _debug_upscale,
)
from .jobs.cutout import run as _generate_cutout_artifacts
from .jobs.depth_plane import run as _generate_depth_plane_geometry
from .jobs.remove_background import run as _remove_background
@@ -58,23 +46,3 @@ def generate_depth_plane_geometry(context, parameters):
@server.job("generate-cutout-artifacts")
def generate_cutout_artifacts(context, parameters):
return _generate_cutout_artifacts(context, parameters)
@server.job("debug-da3")
def debug_da3(context, parameters):
return _debug_da3(context, parameters)
@server.job("debug-moge2")
def debug_moge2(context, parameters):
return _debug_moge2(context, parameters)
@server.job("debug-ben2")
def debug_ben2(context, parameters):
return _debug_ben2(context, parameters)
@server.job("debug-upscale")
def debug_upscale(context, parameters):
return _debug_upscale(context, parameters)

18
src/anyimage/server/geometry/depth_texture.py
View File

@@ -71,12 +71,12 @@ def _write_float_exr(rgba, output_path):
return output_path
def write_vector_depth_texture(frame, output_path):
def write_depth_texture(frame, output_path):
"""Write a frame's camera-space XYZ and validity as float RGBA."""
import numpy as np
if frame.points is None:
raise ValueError("Vector depth requires a point field")
raise ValueError("Depth requires a point field")
points = np.asarray(frame.points, dtype=np.float32)
validity = np.asarray(frame.validity, dtype=np.float32)
if not np.isfinite(points).all():
@@ -87,20 +87,6 @@ def write_vector_depth_texture(frame, output_path):
return _write_float_exr(rgba, output_path)
def write_z_depth_texture(frame, output_path):
"""Write a frame's camera Z and validity as float RGBA."""
import numpy as np
depth = np.asarray(frame.depth, dtype=np.float32)
validity = np.asarray(frame.validity, dtype=np.float32)
if not np.isfinite(depth).all():
raise ValueError("Depth field contains non-finite values")
rgba = np.empty((*depth.shape, 4), dtype=np.float32)
rgba[..., :3] = depth[..., None]
rgba[..., 3] = validity
return _write_float_exr(rgba, output_path)
def write_depth_metadata(frame, reference_valid, output_path):
"""Write camera values needed to scale a model-space depth texture."""
import numpy as np

2
src/anyimage/server/geometry/normal_texture.py
View File

@@ -65,5 +65,3 @@ def write_camera_normal_texture(
output_path = Path(output_path)
full.save(output_path)
return output_path

27
src/anyimage/server/geometry/prediction_artifacts.py
View File

@@ -4,8 +4,7 @@ from ..media import DEFAULT_MAX_AI_INPUT_SIZE, limit_image
from ..models import moge2
from .depth_texture import (
write_depth_metadata,
write_vector_depth_texture,
write_z_depth_texture,
write_depth_texture,
)
from .normal_texture import write_camera_normal_texture
from .edge_depth import extend_edge_depth
@@ -57,16 +56,13 @@ def generate_moge2_artifacts(
parameters,
input_path,
*,
depth_mode="NONE",
generate_depth=False,
normal_mode="NONE",
correct_edge_depth=False,
alpha_threshold=0.9,
):
"""Run MoGe-2 once and write only the requested file artifacts."""
depth_mode = str(depth_mode).upper()
normal_mode = str(normal_mode).upper()
if depth_mode not in {"NONE", "VECTOR", "Z"}:
raise ValueError(f"Unsupported depth mode: {depth_mode}")
if normal_mode not in {"NONE", "TANGENT", "OBJECT"}:
raise ValueError(f"Unsupported normal mode: {normal_mode}")
model_path = _limited_model_input(
@@ -79,32 +75,23 @@ def generate_moge2_artifacts(
context.resource("model_manager"),
parameters,
[model_path],
include_points=depth_mode == "VECTOR",
include_points=generate_depth,
cancel_check=context.check_cancelled,
)
if len(frames) != 1:
raise ValueError("Artifact generation requires one geometry frame")
frame = frames[0]
result = {}
if depth_mode != "NONE":
if generate_depth:
context.progress(0.45, "Generating depth texture")
alpha = _image_alpha(input_path, frame.depth.shape)
depth_frame = (
extend_edge_depth(frame, alpha, alpha_threshold)
if correct_edge_depth else frame
)
if depth_mode == "VECTOR":
depth_path = write_vector_depth_texture(
depth_frame,
context.directory / "vector-depth.exr",
)
result["vector_depth"] = depth_path
else:
depth_path = write_z_depth_texture(
depth_frame,
context.directory / "z-depth.exr",
)
result["z_depth"] = depth_path
result["depth"] = write_depth_texture(
depth_frame, context.directory / "depth.exr",
)
result["depth_metadata"] = write_depth_metadata(
depth_frame,
(alpha > 0.0) & (alpha >= alpha_threshold)

8
src/anyimage/server/jobs/cutout.py
View File

@@ -10,16 +10,16 @@ def run(context, parameters):
context.directory.mkdir(parents=True, exist_ok=True)
input_path = Path(parameters["input"])
result = {}
depth_mode = str(parameters.get("depth_mode", "NONE"))
generate_depth = bool(parameters.get("generate_depth", False))
normal_mode = str(parameters.get("normal_mode", "NONE"))
if depth_mode != "NONE" or normal_mode != "NONE":
if generate_depth or normal_mode != "NONE":
artifacts = generate_moge2_artifacts(
context,
parameters,
input_path,
depth_mode=depth_mode,
generate_depth=generate_depth,
normal_mode=normal_mode,
correct_edge_depth=depth_mode.upper() != "NONE",
correct_edge_depth=generate_depth,
alpha_threshold=float(parameters.get("alpha_threshold", 0.9)),
)
result.update({key: path.name for key, path in artifacts.items()})

96
src/anyimage/server/jobs/debug_predictions.py
View File

@@ -1,96 +0,0 @@
from ..media import copy_frames, input_frames
from ..models import da3
from ..models.ben2 import infer_foreground
from ..models.moge2 import infer_one as infer_moge2
from ..models.upscale import infer_one as infer_upscale
from ..outputs.debug_da3 import write_prediction as write_da3_prediction
from ..outputs.debug_moge2 import write_prediction as write_moge2_prediction
def run_da3(context, parameters):
context.progress(0.02, "Preparing DA3 API input")
context.directory.mkdir(parents=True, exist_ok=True)
with input_frames(context, parameters) as images:
prediction = da3.infer(
context.resource("model_manager"),
parameters,
images,
cancel_check=context.check_cancelled,
)
write_da3_prediction(prediction, context.directory / "prediction.npz")
copy_frames(images, context.directory)
context.check_cancelled()
context.progress(1.0, "Saved raw DA3 API prediction")
return {"directory": "."}
def run_moge2(context, parameters):
context.progress(0.02, "Preparing MoGe-2 API input")
context.directory.mkdir(parents=True, exist_ok=True)
with input_frames(context, parameters) as images:
for index, image in enumerate(images):
context.check_cancelled()
prediction = infer_moge2(
context.resource("model_manager"),
parameters,
image,
release_memory=index == len(images) - 1,
)
suffix = "" if len(images) == 1 else f".{index + 1:04d}"
write_moge2_prediction(
prediction,
context.directory / f"prediction{suffix}.npz",
)
copy_frames(images, context.directory)
context.check_cancelled()
context.progress(1.0, "Saved raw MoGe-2 API prediction")
return {"directory": "."}
def run_ben2(context, parameters):
context.progress(0.02, "Preparing BEN2 API input")
context.directory.mkdir(parents=True, exist_ok=True)
model_manager = context.resource("model_manager")
model_directory = model_manager.directory(parameters["model"])
with input_frames(context, parameters) as images:
for index, image in enumerate(images):
context.check_cancelled()
foreground, _load_ms, _inference_ms = infer_foreground(
image,
model_directory,
parameters["device"],
model_cache=model_manager,
cancel_check=context.check_cancelled,
release_memory=index == len(images) - 1,
)
suffix = "" if len(images) == 1 else f".{index + 1:04d}"
foreground.save(context.directory / f"foreground{suffix}.png")
copy_frames(images, context.directory)
context.check_cancelled()
context.progress(1.0, "Saved raw BEN2 API foreground")
return {"directory": "."}
def run_upscale(context, parameters):
context.progress(0.02, "Preparing Real-ESRGAN API input")
context.directory.mkdir(parents=True, exist_ok=True)
with input_frames(context, parameters) as images:
for index, image in enumerate(images):
context.check_cancelled()
prediction = infer_upscale(
context.resource("model_manager"),
parameters,
image,
cancel_check=context.check_cancelled,
release_memory=index == len(images) - 1,
)
suffix = "" if len(images) == 1 else f".{index + 1:04d}"
prediction.save(
context.directory / f"prediction{suffix}.png",
format="PNG",
compress_level=1,
)
copy_frames(images, context.directory)
context.check_cancelled()
context.progress(1.0, "Saved raw Real-ESRGAN API prediction")
return {"directory": "."}

10
src/anyimage/server/jobs/depth_plane.py
View File

@@ -1,23 +1,23 @@
"""Generate model-space Z Depth and preserve color for one complete image."""
"""Generate model-space depth and preserve color for one complete image."""
from ..geometry.prediction_artifacts import generate_moge2_artifacts
from ..media import copy_color_images, input_frames
def run(context, parameters):
context.progress(0.05, "Preparing Depth Plane")
context.progress(0.05, "Preparing depth conversion")
context.directory.mkdir(parents=True, exist_ok=True)
with input_frames(context, parameters) as images:
if len(images) != 1:
raise ValueError("Depth Plane currently supports one image")
raise ValueError("Depth conversion currently supports one image")
copy_color_images(images, context.directory)
artifacts = generate_moge2_artifacts(
context,
parameters,
images[0],
depth_mode="Z",
generate_depth=True,
normal_mode="TANGENT",
)
context.check_cancelled()
context.progress(1.0, "Depth Plane data is ready")
context.progress(1.0, "Depth conversion data is ready")
return {key: path.name for key, path in artifacts.items()}

1
src/anyimage/server/jobs/upscale.py
View File

@@ -29,7 +29,6 @@ def run(context, parameters):
parameters,
image,
cancel_check=context.check_cancelled,
preserve_alpha=True,
release_memory=index == total - 1,
)
prediction.save(

2
src/anyimage/server/media/__init__.py
View File

@@ -6,7 +6,6 @@ from .input import (
check_cancelled,
collect_images,
copy_color_images,
copy_frames,
extract_video_frames,
first_frames,
input_frames,
@@ -31,7 +30,6 @@ __all__ = (
"check_cancelled",
"collect_images",
"copy_color_images",
"copy_frames",
"extract_video_frames",
"first_frames",
"input_frames",

13
src/anyimage/server/media/input.py
View File

@@ -157,16 +157,3 @@ def copy_color_images(images, output_directory):
if Path(image_path).resolve() == destination.resolve():
continue
shutil.copy2(image_path, destination)
def copy_frames(images, output_directory):
"""Copy multiple input images as numbered raw Debug frame files."""
if len(images) <= 1:
return
for index, image_path in enumerate(images):
suffix = Path(image_path).suffix.lower() or ".jpg"
destination = Path(output_directory) / f"frame.{index + 1:04d}{suffix}"
if Path(image_path).resolve() == destination.resolve():
continue
shutil.copy2(image_path, destination)

80
src/anyimage/server/model_catalog.py
View File

@@ -9,70 +9,13 @@ class ModelSpec:
repository: str
license_name: str
description: str
supports_gs: bool
huggingface_repository: str = ""
download_patterns: tuple[str, ...] = ()
ready_patterns: tuple[str, ...] = ("config.json", "*.safetensors")
enabled: bool = True
r2_directory: str = ""
r2_files: tuple[tuple[str, int, str], ...] = ()
MODELS = {
"SMALL": ModelSpec(
key="SMALL",
label="DA3 Small",
directory_name="DA3-SMALL",
repository="depth-anything/DA3-SMALL",
license_name="Apache 2.0",
huggingface_repository="depth-anything/DA3-SMALL",
description="Fastest any-view model using ONNX Runtime",
supports_gs=False,
download_patterns=("model.onnx",),
ready_patterns=("model.onnx",),
r2_directory="DA3-SMALL",
r2_files=(
("model.onnx", 105484029, "7a9d81475b3e2334ace0d6d2c2a816b3c93081437e1866cc434ce1789125f631"),
),
),
"BASE": ModelSpec(
key="BASE",
label="DA3 Base",
directory_name="DA3-BASE",
repository="depth-anything/DA3-BASE",
license_name="Apache 2.0",
huggingface_repository="depth-anything/DA3-BASE",
description="Balanced any-view model using ONNX Runtime",
supports_gs=False,
download_patterns=("model.onnx",),
ready_patterns=("model.onnx",),
r2_directory="DA3-BASE",
r2_files=(
("model.onnx", 412856697, "31f38fc4990539d7902ba267924b9731db1acb00d840c248a60b1865810ff9e6"),
),
),
"GIANT_1_1": ModelSpec(
key="GIANT_1_1",
label="DA3 Giant 1.1",
directory_name="da3_giant_1_1",
repository="depth-anything/DA3-GIANT-1.1",
license_name="CC BY-NC 4.0",
description="Non-commercial Giant model with Gaussian head",
supports_gs=True,
enabled=False,
),
"NESTED_GIANT_LARGE_1_1": ModelSpec(
key="NESTED_GIANT_LARGE_1_1",
label="DA3 Nested Giant Large 1.1",
directory_name="da3_nested_giant_large_1_1",
repository="depth-anything/DA3NESTED-GIANT-LARGE-1.1",
license_name="CC BY-NC 4.0",
description="Non-commercial nested model with metric depth and Gaussian head",
supports_gs=True,
enabled=False,
),
}
BEN2_MODEL = ModelSpec(
key="BEN2_BASE",
label="BEN2 Base",
@@ -81,7 +24,6 @@ BEN2_MODEL = ModelSpec(
license_name="Apache 2.0",
huggingface_repository="onnx-community/BEN2-ONNX",
description="Fast high-quality FP16 ONNX background removal model",
supports_gs=False,
download_patterns=("onnx/model_fp16.onnx",),
ready_patterns=("onnx/model_fp16.onnx",),
r2_directory="BEN2-ONNX",
@@ -100,7 +42,6 @@ REALESRGAN_GENERAL_WDN_X4V3 = ModelSpec(
repository="xinntao/Real-ESRGAN",
license_name="BSD 3-Clause",
description="Fast and conservative 2x output from a 4x ONNX model",
supports_gs=False,
download_patterns=("realesr-general-wdn-x4v3.onnx",),
ready_patterns=("realesr-general-wdn-x4v3.onnx",),
r2_directory="realesr-general-wdn-x4v3",
@@ -118,7 +59,6 @@ SAFMN_MODEL = ModelSpec(
repository="sunny2109/SAFMN",
license_name="Apache 2.0",
description="Balanced 2x output from a real-world 4x ONNX model",
supports_gs=False,
download_patterns=("safmn-real-x4.onnx",),
ready_patterns=("safmn-real-x4.onnx",),
r2_directory="safmn-real-x4",
@@ -137,7 +77,6 @@ REALESRGAN_X4PLUS = ModelSpec(
license_name="BSD 3-Clause",
huggingface_repository="jonathanst29/tinier-upscale-models",
description="Detailed 2x output from a general-purpose 4x ONNX model",
supports_gs=False,
download_patterns=("realesrgan-x4plus.onnx",),
ready_patterns=("realesrgan-x4plus.onnx",),
r2_directory="realesrgan-x4plus",
@@ -156,7 +95,6 @@ PISA_SR = ModelSpec(
repository="csslc/PiSA-SR",
license_name="Apache 2.0 / CreativeML Open RAIL++-M",
description="Generative 2x upscale with fixed detail strength 0.7",
supports_gs=False,
download_patterns=("encoder.onnx", "decoder.onnx", "unet-pixel.onnx", "unet-full.onnx", "noise.npy", "LICENSE-APACHE.txt", "LICENSE-MODEL.txt", "NOTICE.txt"),
ready_patterns=("encoder.onnx", "decoder.onnx", "unet-pixel.onnx", "unet-full.onnx", "noise.npy", "LICENSE-APACHE.txt", "LICENSE-MODEL.txt", "NOTICE.txt"),
r2_directory="pisa-sr",
@@ -196,7 +134,6 @@ MOGE2_MODELS = {
license_name="MIT",
huggingface_repository="Ruicheng/moge-2-vits-normal-onnx",
description="Fastest MoGe-2 model using ONNX Runtime",
supports_gs=False,
download_patterns=("model.onnx",),
ready_patterns=("model.onnx",),
r2_directory="moge-2-vits-normal-onnx",
@@ -214,7 +151,6 @@ MOGE2_MODELS = {
license_name="MIT",
huggingface_repository="Ruicheng/moge-2-vitb-normal-onnx",
description="Balanced MoGe-2 model using ONNX Runtime",
supports_gs=False,
download_patterns=("model.onnx",),
ready_patterns=("model.onnx",),
r2_directory="moge-2-vitb-normal-onnx",
@@ -232,7 +168,6 @@ MOGE2_MODELS = {
license_name="MIT",
huggingface_repository="Ruicheng/moge-2-vitl-normal-onnx",
description="Highest-quality MoGe-2 model using ONNX Runtime",
supports_gs=False,
download_patterns=("model.onnx",),
ready_patterns=("model.onnx",),
r2_directory="moge-2-vitl-normal-onnx",
@@ -245,21 +180,10 @@ MOGE2_MODELS = {
}
MOGE2_MODEL = MOGE2_MODELS["MOGE2_VITS_NORMAL"]
DOWNLOADABLE_MODELS = {
**MODELS,
BEN2_MODEL.key: BEN2_MODEL,
**UPSCALE_MODELS,
**MOGE2_MODELS,
}
MODEL_NUMBERS = {
model_key: number for number, model_key in enumerate(MODELS)
}
def get_model(model_key):
try:
return MODELS[model_key]
except KeyError as error:
raise ValueError(f"Unknown DA3 model: {model_key}") from error
def get_downloadable_model(model_key):
@@ -281,16 +205,12 @@ def model_record(model, ready=False):
"label": model.label,
"description": model.description,
"license": model.license_name,
"supports_gs": model.supports_gs,
"enabled": model.enabled,
"family": (
"moge2"
if model.key.startswith("MOGE2_")
else "ben2"
if model.key == BEN2_MODEL.key
else "upscale"
if model.key in UPSCALE_MODELS
else "da3"
),
"ready": bool(ready),
}

25
src/anyimage/server/model_manager.py
View File

@@ -14,8 +14,6 @@ class ModelSessionCache:
def __init__(self):
self.ben2_model = None
self.ben2_key = None
self.da3_model = None
self.da3_key = None
self.moge_model = None
self.moge_key = None
self.upscale_model = None
@@ -39,21 +37,6 @@ class ModelSessionCache:
elapsed_ms = (time.perf_counter() - started) * 1000.0
return model, elapsed_ms
def get_da3(self, model_dir, device):
requested_device = getattr(device, "type", device)
key = (str(Path(model_dir).resolve()), requested_device, "onnx")
if self.da3_model is not None and self.da3_key == key:
return self.da3_model
self.release_geometry()
return self._load_da3(model_dir, requested_device, key)
def _load_da3(self, model_dir, requested_device, key):
from .models.onnx_da3 import create_session
model = create_session(model_dir, requested_device)
self.da3_model = model
self.da3_key = key
return model
def get_moge2(self, model_dir, requested_device):
from .models.onnx_moge2 import create_session
@@ -88,8 +71,6 @@ class ModelSessionCache:
self._release_runtime_cache()
def release_geometry(self):
self.da3_model = None
self.da3_key = None
self.moge_model = None
self.moge_key = None
self._release_runtime_cache()
@@ -107,17 +88,15 @@ class ModelSessionCache:
if close is not None:
close()
self.ben2_model = None
self.da3_model = None
self.moge_model = None
self.upscale_model = None
self.ben2_key = None
self.da3_key = None
self.moge_key = None
self.upscale_key = None
self._release_runtime_cache()
def snapshot(self):
geometry_key = self.da3_key or self.moge_key
geometry_key = self.moge_key
return {
"ben2": Path(self.ben2_key[0]).name if self.ben2_key else "",
"geometry": Path(geometry_key[0]).name if geometry_key else "",
@@ -169,8 +148,6 @@ class ModelManager:
def get_ben2(self, model_dir, device):
return self.sessions.get_ben2(model_dir, device)
def get_da3(self, model_dir, device):
return self.sessions.get_da3(model_dir, device)
def get_moge2(self, model_dir, device):
return self.sessions.get_moge2(model_dir, device)

3
src/anyimage/server/models/__init__.py
View File

@@ -1,8 +1,7 @@
from . import ben2, da3, moge2, upscale
from . import ben2, moge2, upscale
__all__ = (
"ben2",
"da3",
"moge2",
"upscale",
)

109
src/anyimage/server/models/da3.py
View File

@@ -1,109 +0,0 @@
from ..geometry import unproject_camera_depth
from .geometry import GeometryFrame
def merge_frame_predictions(predictions):
import numpy as np
if not predictions:
raise ValueError("DA3 returned no frame predictions")
merged = predictions[0]
for name in (
"depth",
"conf",
"extrinsics",
"intrinsics",
"processed_images",
):
setattr(
merged,
name,
np.concatenate(
[getattr(prediction, name) for prediction in predictions],
axis=0,
),
)
return merged
def infer(
model_manager,
parameters,
images,
*,
progress_callback=None,
cancel_check=None,
):
from .onnx_da3 import infer as infer_onnx
from .onnx_runtime import provider_label
model_directory = model_manager.directory(parameters["model"])
device = parameters["device"]
model = model_manager.get_da3(model_directory, device)
options = {
"process_res": parameters["process_res"],
"process_res_method": parameters["process_res_method"],
"ref_view_strategy": parameters["ref_view_strategy"],
}
def run(paths, *, release_memory):
try:
return infer_onnx(
model,
paths,
**options,
release_memory=release_memory,
)
except UnicodeDecodeError as error:
if provider_label(model) == "DirectML":
raise RuntimeError(
"DA3 inference failed on DirectML. Switch the device to "
"CPU in AnyImage Preferences and retry. "
f"Original error: {error}"
) from error
raise
if parameters.get("inference_mode", "joint") != "per_frame":
return run(
[str(path) for path in images],
release_memory=True,
)
predictions = []
for index, image in enumerate(images):
if cancel_check is not None:
cancel_check()
if progress_callback is not None:
progress_callback(index, len(images))
predictions.append(
run(
[str(image)],
release_memory=index == len(images) - 1,
)
)
return merge_frame_predictions(predictions)
def geometry_frames(prediction):
import numpy as np
depth = np.asarray(prediction.depth, dtype=np.float32)
validity = np.asarray(prediction.conf, dtype=np.float32)
intrinsics = np.asarray(prediction.intrinsics, dtype=np.float32)
if validity.shape != depth.shape or intrinsics.shape != (len(depth), 3, 3):
raise ValueError("DA3 prediction fields do not share one frame layout")
return [
GeometryFrame(
depth=frame_depth,
validity=frame_validity,
intrinsics=frame_intrinsics,
points=unproject_camera_depth(frame_depth, frame_intrinsics).astype(
np.float32
),
)
for frame_depth, frame_validity, frame_intrinsics in zip(
depth,
validity,
intrinsics,
)
]

1
src/anyimage/server/models/onnx_ben2.py
View File

@@ -55,4 +55,3 @@ def infer(session, image, *, release_memory=True):
foreground = image.convert("RGBA")
foreground.putalpha(mask)
return foreground

246
src/anyimage/server/models/onnx_da3.py
View File

@@ -1,246 +0,0 @@
"""Depth Anything 3 inference with ONNX Runtime and NumPy."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
import numpy as np
try:
from .onnx_runtime import create_session as create_onnx_session
from .onnx_runtime import run_session
except ImportError:
from onnx_runtime import create_session as create_onnx_session
from onnx_runtime import run_session
ONNX_PATH = "model.onnx"
PATCH_SIZE = 14
# DA3 cross-view attention is quadratic in (views x patches); the peak
# attention tensor alone can exhaust memory long before ONNX Runtime reports
# a generic allocation failure.
ATTENTION_BUDGET_BYTES = 4 * 1024 * 1024 * 1024
STRATEGY_CODES = {
"first": 0,
"middle": 1,
"saddle_balanced": 2,
"saddle_sim_range": 3,
}
IMAGE_MEAN = np.asarray((0.485, 0.456, 0.406), dtype=np.float32)
IMAGE_STD = np.asarray((0.229, 0.224, 0.225), dtype=np.float32)
@dataclass
class Prediction:
depth: np.ndarray
conf: np.ndarray
extrinsics: np.ndarray
intrinsics: np.ndarray
processed_images: np.ndarray
is_metric: int = 0
sky: np.ndarray | None = None
gaussians: object | None = None
aux: dict | None = None
scale_factor: float | None = None
def create_session(model_dir, requested_device="auto"):
path = Path(model_dir) / ONNX_PATH
return create_onnx_session(path, requested_device)
def nearest_multiple(value, multiple=PATCH_SIZE):
lower = value // multiple * multiple
upper = lower + multiple
return upper if abs(upper - value) <= abs(value - lower) else lower
def target_size(size, process_res, method):
width, height = size
boundary = max(width, height)
if method.startswith("lower_bound"):
boundary = min(width, height)
scale = float(process_res) / boundary
width = max(1, int(round(width * scale)))
height = max(1, int(round(height * scale)))
if method.endswith("resize"):
return (
max(PATCH_SIZE, nearest_multiple(width)),
max(PATCH_SIZE, nearest_multiple(height)),
)
if method.endswith("crop"):
return (
max(PATCH_SIZE, width // PATCH_SIZE * PATCH_SIZE),
max(PATCH_SIZE, height // PATCH_SIZE * PATCH_SIZE),
)
raise ValueError(f"Unsupported process_res_method: {method}")
def resize_and_crop(image, process_res, method):
from PIL import Image
width, height = image.size
boundary = max(width, height)
if method.startswith("lower_bound"):
boundary = min(width, height)
scale = float(process_res) / boundary
resized_size = (
max(1, int(round(width * scale))),
max(1, int(round(height * scale))),
)
resampling = (
Image.Resampling.BICUBIC
if scale > 1.0
else Image.Resampling.BOX
)
image = image.resize(resized_size, resample=resampling)
final_size = target_size((width, height), process_res, method)
if method.endswith("resize"):
if image.size != final_size:
upscale = final_size[0] > image.width or final_size[1] > image.height
resampling = (
Image.Resampling.BICUBIC
if upscale
else Image.Resampling.BOX
)
image = image.resize(final_size, resample=resampling)
return image
left = (image.width - final_size[0]) // 2
top = (image.height - final_size[1]) // 2
return image.crop((left, top, left + final_size[0], top + final_size[1]))
def preprocess(image_paths, process_res=504, process_res_method="upper_bound_resize"):
from PIL import Image
processed = []
for path in image_paths:
with Image.open(path) as source:
image = resize_and_crop(
source.convert("RGB"),
process_res,
process_res_method,
)
processed.append(np.asarray(image, dtype=np.uint8))
min_height = min(image.shape[0] for image in processed)
min_width = min(image.shape[1] for image in processed)
cropped = []
for image in processed:
top = (image.shape[0] - min_height) // 2
left = (image.shape[1] - min_width) // 2
cropped.append(
image[top : top + min_height, left : left + min_width]
)
images = np.stack(cropped)
normalized = images.astype(np.float32) / 255.0
normalized = (normalized - IMAGE_MEAN) / IMAGE_STD
model_input = normalized.transpose(0, 3, 1, 2)[None]
return np.ascontiguousarray(model_input), images
def quaternion_to_matrix(quaternion):
i, j, k, real = np.moveaxis(quaternion, -1, 0)
two_scale = 2.0 / np.sum(quaternion * quaternion, axis=-1)
values = (
1 - two_scale * (j * j + k * k),
two_scale * (i * j - k * real),
two_scale * (i * k + j * real),
two_scale * (i * j + k * real),
1 - two_scale * (i * i + k * k),
two_scale * (j * k - i * real),
two_scale * (i * k - j * real),
two_scale * (j * k + i * real),
1 - two_scale * (i * i + j * j),
)
return np.stack(values, axis=-1).reshape(quaternion.shape[:-1] + (3, 3))
def camera_from_pose(pose_encoding, height, width):
translation = pose_encoding[..., :3]
rotation = quaternion_to_matrix(pose_encoding[..., 3:7])
rotation_inverse = np.swapaxes(rotation, -1, -2)
translation_inverse = -rotation_inverse @ translation[..., None]
extrinsics = np.concatenate(
(rotation_inverse, translation_inverse),
axis=-1,
).astype(np.float32)
field_of_view_height = pose_encoding[..., 7]
field_of_view_width = pose_encoding[..., 8]
focal_y = (height / 2.0) / np.maximum(
np.tan(field_of_view_height / 2.0),
1e-6,
)
focal_x = (width / 2.0) / np.maximum(
np.tan(field_of_view_width / 2.0),
1e-6,
)
intrinsics = np.zeros(pose_encoding.shape[:-1] + (3, 3), dtype=np.float32)
intrinsics[..., 0, 0] = focal_x
intrinsics[..., 1, 1] = focal_y
intrinsics[..., 0, 2] = width / 2.0
intrinsics[..., 1, 2] = height / 2.0
intrinsics[..., 2, 2] = 1.0
return extrinsics, intrinsics
def estimate_attention_bytes(views, height, width):
"""Estimate the peak cross-view attention tensor size in bytes."""
patches = (height // PATCH_SIZE) * (width // PATCH_SIZE)
tokens = views * patches
# 12 attention heads in DA3 Base; float32 activations.
return tokens * tokens * 12 * 4
def infer(
session,
image_paths,
process_res=504,
process_res_method="upper_bound_resize",
ref_view_strategy="saddle_balanced",
*,
release_memory=True,
):
inputs = session.get_inputs()
model_input, processed_images = preprocess(
image_paths,
process_res,
process_res_method,
)
_, views, _, height, width = model_input.shape
attention_bytes = estimate_attention_bytes(views, height, width)
if attention_bytes > ATTENTION_BUDGET_BYTES:
raise RuntimeError(
f"DA3 cannot process {views} frame(s) at {width}x{height}: "
"the cross-view attention would need about "
f"{attention_bytes // 2**30} GB. Reduce the frame count or "
"the Resolution Scale."
)
strategy_code = np.asarray(
[STRATEGY_CODES.get(ref_view_strategy, 2)],
dtype=np.int64,
)
input_names = [value.name for value in inputs]
outputs = run_session(
session,
None,
{
input_names[0]: model_input,
input_names[1]: strategy_code,
},
release_memory=release_memory,
)
depth, confidence, pose_encoding = outputs[:3]
height, width = depth.shape[-2:]
extrinsics, intrinsics = camera_from_pose(
pose_encoding[0],
height,
width,
)
return Prediction(
depth=np.asarray(depth[0], dtype=np.float32),
conf=np.asarray(confidence[0], dtype=np.float32),
extrinsics=extrinsics,
intrinsics=intrinsics,
processed_images=processed_images,
aux={},
)

2
src/anyimage/server/models/onnx_upscale.py
View File

@@ -153,5 +153,3 @@ def infer(session, image, cancel_check=None, *, release_memory=True):
raise UpscaleResourceError(
"the complete upscale result exceeded available system memory"
) from error

3
src/anyimage/server/models/upscale.py
View File

@@ -12,7 +12,6 @@ def infer_one(
parameters,
image_path,
cancel_check=None,
preserve_alpha=False,
release_memory=True,
):
from PIL import Image
@@ -33,7 +32,7 @@ def infer_one(
opened,
parameters.get("max_input_size", DEFAULT_MAX_AI_INPUT_SIZE),
)
source_alpha = _source_alpha(source) if preserve_alpha else None
source_alpha = _source_alpha(source)
prediction = infer(
model,
source,

1
src/anyimage/server/outputs/__init__.py
View File

@@ -1 +0,0 @@
"""Debug output serializers used by Job Server workflows."""

24
src/anyimage/server/outputs/debug_da3.py
View File

@@ -1,24 +0,0 @@
from pathlib import Path
def write_prediction(prediction, output_path):
import numpy as np
values = {}
for name in (
"depth",
"conf",
"extrinsics",
"intrinsics",
"processed_images",
"is_metric",
"sky",
"scale_factor",
):
value = getattr(prediction, name, None)
if value is not None:
values[name] = np.asarray(value)
output_path = Path(output_path)
output_path.parent.mkdir(parents=True, exist_ok=True)
np.savez(output_path, **values)
return output_path

13
src/anyimage/server/outputs/debug_moge2.py
View File

@@ -1,13 +0,0 @@
from pathlib import Path
def write_prediction(prediction, output_path):
import numpy as np
output_path = Path(output_path)
output_path.parent.mkdir(parents=True, exist_ok=True)
np.savez(
output_path,
**{name: np.asarray(value) for name, value in prediction.items()},
)
return output_path