feat(cutout_tool): Implement cutout region to shape functionality
- Added interaction helpers for cutout region validation and shape menu interaction. - Created main operator for converting image regions into cutout shapes, including AI artifact generation. - Introduced object creation logic for cutout shapes, handling depth and normal textures. - Defined cutout shape identifiers and node groups for various shape types. - Implemented depth texture writing and prediction artifact generation for MoGe-2. - Developed server jobs for generating cutout artifacts, depth planes, and refining image regions. - Added region refinement logic to enhance gesture masks using BEN2.
This commit is contained in:
292
src/anyimage/operators/cutout_tool/object.py
Normal file
292
src/anyimage/operators/cutout_tool/object.py
Normal file
@@ -0,0 +1,292 @@
|
||||
"""Create Blender data for a Cutout Shape."""
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
import bpy
|
||||
|
||||
from ..common.image_data import (
|
||||
color_data_name,
|
||||
image_rgba,
|
||||
)
|
||||
from ..common.depth_data import camera_depth_values, depth_uniform_scale
|
||||
from ..common.mesh_result import (
|
||||
create_image_material,
|
||||
finalize_object_result,
|
||||
load_asset_node_group,
|
||||
modifier_input_identifier,
|
||||
set_modifier_input,
|
||||
)
|
||||
from .geometry import (
|
||||
DEFAULT_DEPTH_BASELINE,
|
||||
DEFAULT_DEPTH_DIRECTION,
|
||||
crop_plane_bounds,
|
||||
cutout_cleanup_value,
|
||||
cutout_point_spacing,
|
||||
depth_balloon_projection_parameters,
|
||||
metric_depth_reference,
|
||||
region_calibration,
|
||||
use_adaptive_depth_calibration,
|
||||
world_plane_metric,
|
||||
)
|
||||
from .shape import (
|
||||
BALLOON_ATTRIBUTE_NAME,
|
||||
CUTOUT_NODE_GROUP_NAME,
|
||||
CUTOUT_SHAPE_INDICES,
|
||||
DEPTH_CUTOUT_SHAPES,
|
||||
)
|
||||
|
||||
|
||||
class NormalMode(StrEnum):
|
||||
NONE = "NONE"
|
||||
TANGENT = "TANGENT"
|
||||
OBJECT = "OBJECT"
|
||||
|
||||
|
||||
CUTOUT_AXIS_TO_SOURCE = (
|
||||
(0.0, -1.0, 0.0, 0.0),
|
||||
(0.0, 0.0, 1.0, 0.0),
|
||||
(-1.0, 0.0, 0.0, 0.0),
|
||||
(0.0, 0.0, 0.0, 1.0),
|
||||
)
|
||||
|
||||
|
||||
def _cutout_world_matrix(source_matrix, crop_center):
|
||||
from mathutils import Matrix
|
||||
|
||||
center_x, center_y = (float(value) for value in crop_center)
|
||||
return (
|
||||
source_matrix
|
||||
@ Matrix.Translation((center_x, center_y, 0.0))
|
||||
@ Matrix(CUTOUT_AXIS_TO_SOURCE)
|
||||
)
|
||||
|
||||
|
||||
def _assign_uv(mesh, uv):
|
||||
import numpy as np
|
||||
|
||||
for layer in list(mesh.uv_layers):
|
||||
mesh.uv_layers.remove(layer)
|
||||
uv_layer = mesh.uv_layers.new(name="UVMap")
|
||||
loop_vertices = np.empty(len(mesh.loops), dtype=np.int32)
|
||||
mesh.loops.foreach_get("vertex_index", loop_vertices)
|
||||
loop_uv = np.asarray(uv, dtype=np.float32)[loop_vertices]
|
||||
uv_layer.data.foreach_set("uv", loop_uv.ravel())
|
||||
|
||||
|
||||
def _assign_float_point_attribute(mesh, name, values):
|
||||
import numpy as np
|
||||
|
||||
attribute = mesh.attributes.new(name=name, type="FLOAT", domain="POINT")
|
||||
attribute.data.foreach_set("value", np.asarray(values, dtype=np.float32))
|
||||
|
||||
|
||||
def _cropped_color_image(source_object, region_mask, source_rgba):
|
||||
crop_bounds = region_mask.bounds
|
||||
import numpy as np
|
||||
|
||||
source = source_object.data
|
||||
rgba = np.asarray(source_rgba)
|
||||
left, top, right, bottom = crop_bounds
|
||||
width = right - left
|
||||
height = bottom - top
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("The Cutout crop dimensions must be positive")
|
||||
cropped = np.zeros((height, width, 4), dtype=np.float32)
|
||||
source_height, source_width = rgba.shape[:2]
|
||||
source_left = max(0, left)
|
||||
source_top = max(0, top)
|
||||
source_right = min(source_width, right)
|
||||
source_bottom = min(source_height, bottom)
|
||||
if source_right > source_left and source_bottom > source_top:
|
||||
cropped[
|
||||
source_top - top : source_bottom - top,
|
||||
source_left - left : source_right - left,
|
||||
] = rgba[source_top:source_bottom, source_left:source_right]
|
||||
image = bpy.data.images.new(
|
||||
color_data_name(source_object),
|
||||
width=width,
|
||||
height=height,
|
||||
alpha=True,
|
||||
)
|
||||
try:
|
||||
image.colorspace_settings.name = source.colorspace_settings.name
|
||||
image.alpha_mode = source.alpha_mode
|
||||
image.pixels.foreach_set(np.flipud(cropped).ravel())
|
||||
image.pack()
|
||||
image.update()
|
||||
except Exception:
|
||||
bpy.data.images.remove(image, do_unlink=True)
|
||||
raise
|
||||
return image
|
||||
|
||||
|
||||
def load_cutout_node_group():
|
||||
return load_asset_node_group(CUTOUT_NODE_GROUP_NAME)
|
||||
|
||||
|
||||
def create_shape_object(
|
||||
context,
|
||||
source_object,
|
||||
shape,
|
||||
mesh_resolution,
|
||||
region_mask,
|
||||
base_shape,
|
||||
*,
|
||||
depth_image=None,
|
||||
depth_metadata=None,
|
||||
normal_image=None,
|
||||
normal_mode=NormalMode.NONE,
|
||||
source_rgba=None,
|
||||
):
|
||||
import numpy as np
|
||||
|
||||
normal_mode = NormalMode(normal_mode)
|
||||
bounds = crop_plane_bounds(
|
||||
source_object,
|
||||
region_mask.bounds,
|
||||
tuple(source_object.data.size),
|
||||
)
|
||||
crop_center = (
|
||||
(bounds[0] + bounds[1]) * 0.5,
|
||||
(bounds[2] + bounds[3]) * 0.5,
|
||||
)
|
||||
world_metric, _world_z_scale = world_plane_metric(source_object.matrix_world)
|
||||
cleanup_value = cutout_cleanup_value(
|
||||
cutout_point_spacing(source_object, mesh_resolution),
|
||||
world_metric,
|
||||
)
|
||||
color_image = _cropped_color_image(
|
||||
source_object,
|
||||
region_mask,
|
||||
image_rgba(source_object.data) if source_rgba is None else source_rgba,
|
||||
)
|
||||
material = None
|
||||
mesh = None
|
||||
mesh_object = None
|
||||
try:
|
||||
material = create_image_material(
|
||||
source_object,
|
||||
color_image,
|
||||
0.0,
|
||||
normal_image=normal_image,
|
||||
normal_space=(
|
||||
NormalMode.TANGENT.value
|
||||
if normal_mode == NormalMode.NONE
|
||||
else normal_mode.value
|
||||
),
|
||||
ior=1.2,
|
||||
scene=context.scene,
|
||||
)
|
||||
depth_reference = None
|
||||
uniform_scale = None
|
||||
depth_direction = None
|
||||
if shape in DEPTH_CUTOUT_SHAPES:
|
||||
if depth_image is None or depth_metadata is None:
|
||||
raise ValueError("Depth Shape requires a Depth texture and metadata")
|
||||
camera_depth, valid = camera_depth_values(depth_image)
|
||||
calibration_valid, transparent_fraction = region_calibration(
|
||||
region_mask, valid.shape
|
||||
)
|
||||
calibration_valid &= valid
|
||||
uniform_scale = depth_uniform_scale(
|
||||
source_object,
|
||||
depth_metadata,
|
||||
bounds,
|
||||
)
|
||||
if not use_adaptive_depth_calibration(transparent_fraction):
|
||||
depth_reference = DEFAULT_DEPTH_BASELINE
|
||||
if shape == "DEPTH_BALLOON":
|
||||
depth_direction = DEFAULT_DEPTH_DIRECTION
|
||||
elif shape == "DEPTH_SURFACE":
|
||||
depth_reference = metric_depth_reference(
|
||||
camera_depth,
|
||||
calibration_valid,
|
||||
uniform_scale,
|
||||
)
|
||||
else:
|
||||
depth_reference, depth_direction = depth_balloon_projection_parameters(
|
||||
base_shape.vertices,
|
||||
base_shape.uv,
|
||||
camera_depth,
|
||||
calibration_valid,
|
||||
uniform_scale,
|
||||
)
|
||||
|
||||
source_name = source_object.name
|
||||
mesh = bpy.data.meshes.new(source_name)
|
||||
mesh.from_pydata(
|
||||
np.asarray(base_shape.vertices).tolist(),
|
||||
[],
|
||||
base_shape.faces,
|
||||
)
|
||||
for face in mesh.polygons:
|
||||
face.use_smooth = True
|
||||
_assign_uv(mesh, base_shape.uv)
|
||||
_assign_float_point_attribute(
|
||||
mesh,
|
||||
BALLOON_ATTRIBUTE_NAME,
|
||||
base_shape.balloon,
|
||||
)
|
||||
mesh.materials.append(material)
|
||||
mesh.update()
|
||||
|
||||
mesh_object = bpy.data.objects.new(source_name, mesh)
|
||||
context.collection.objects.link(mesh_object)
|
||||
mesh_object.matrix_world = _cutout_world_matrix(
|
||||
source_object.matrix_world,
|
||||
crop_center,
|
||||
)
|
||||
mesh_object["anyimage_mesh_shape"] = shape
|
||||
node_group = load_cutout_node_group()
|
||||
modifier = mesh_object.modifiers.new(name=node_group.name, type="NODES")
|
||||
modifier.node_group = node_group
|
||||
modifier.show_group_selector = False
|
||||
for name, value in (
|
||||
("Shape", CUTOUT_SHAPE_INDICES[shape]),
|
||||
("Cleanup", cleanup_value),
|
||||
("Smooth Iterations", 0),
|
||||
("Smooth", 1.0),
|
||||
(
|
||||
"Thickness",
|
||||
1.0 if shape in {"BALLOON", "DEPTH_BALLOON"} else 0.0,
|
||||
),
|
||||
):
|
||||
set_modifier_input(
|
||||
modifier,
|
||||
modifier_input_identifier(node_group, name),
|
||||
value,
|
||||
)
|
||||
if shape in DEPTH_CUTOUT_SHAPES:
|
||||
for name, value in (
|
||||
("Depth Scale", 1.0),
|
||||
("Uniform Scale", uniform_scale),
|
||||
("Base Plane Depth", depth_reference),
|
||||
("Depth Image", depth_image),
|
||||
):
|
||||
set_modifier_input(
|
||||
modifier,
|
||||
modifier_input_identifier(node_group, name),
|
||||
value,
|
||||
)
|
||||
if shape == "DEPTH_BALLOON":
|
||||
set_modifier_input(
|
||||
modifier,
|
||||
modifier_input_identifier(node_group, "Depth Direction"),
|
||||
depth_direction,
|
||||
)
|
||||
return finalize_object_result(
|
||||
context,
|
||||
source_object,
|
||||
mesh_object,
|
||||
keep_source=True,
|
||||
)
|
||||
except Exception:
|
||||
if mesh_object is not None:
|
||||
bpy.data.objects.remove(mesh_object, do_unlink=True)
|
||||
if mesh is not None and mesh.users == 0:
|
||||
bpy.data.meshes.remove(mesh)
|
||||
if material is not None and material.users == 0:
|
||||
bpy.data.materials.remove(material)
|
||||
if color_image.users == 0:
|
||||
bpy.data.images.remove(color_image, do_unlink=True)
|
||||
raise
|
||||
Reference in New Issue
Block a user