icrdr/AnyImage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement cutout tool and depth generation features

Browse Source 
- Added a new cutout tool for selecting regions in images, allowing users to choose mesh shapes based on selected areas.
- Introduced depth generation capabilities for rasterized images, including normal data extraction.
- Created jobs for generating depth planes and region data, enhancing the AI data generation workflow.
- Developed a selection refinement process for cutout regions, improving accuracy in generated outputs.
- Established a new region module for handling rasterized image regions and their associated AI generation tasks.
- Enhanced node graph builders for asset generation, facilitating smoother integration of new features.
This commit is contained in:
马南
2026-08-30 23:09:13 +08:00
parent 32bcd45d7b
commit 08309f719d
88 changed files with 4145 additions and 5639 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/anyimage/operators/cutout_tool/__init__.py
View File

@@ -1,12 +1,8 @@
"""Cutout selection, shape conversion, and geometry algorithms."""
from .selection_tool import ActivateCutoutTool, CutoutTool, SelectCutoutRegion
from .shape_conversion import ConvertCutoutShape, GenerateCutoutShape
from .region_to_shape import CutoutRegionToShape
from .select_region import CutoutTool, SelectCutoutRegion
__all__ = (
"ActivateCutoutTool",
"ConvertCutoutShape",
"CutoutRegionToShape",
"CutoutTool",
"GenerateCutoutShape",
"SelectCutoutRegion",
)

814
src/anyimage/operators/cutout_tool/build_shapes.py
View File

@@ -1,814 +0,0 @@
import json
import bpy
from ...preferences import addon_preferences, configured_max_ai_input_size
from ...server.media.resolution import limited_image_size
from ..common.image import (
color_data_name,
create_color_image,
depth_data_name,
is_image_empty,
material_data_name,
normal_data_name,
plane_bounds,
video_total_frames,
)
from ..common.material import create_material
from ..common.node_asset import modifier_input_identifier, set_modifier_input
from ..common.normal_map import bake_tangent_normal_image
from ..common.operator import (
push_undo,
)
from .depth_calibration import (
DEFAULT_DEPTH_DIRECTION,
UNREFINED_DEPTH_REFERENCE,
depth_balloon_projection_parameters,
metric_depth_reference,
)
from .planar_mesh import (
metric_transform,
prepare_planar_regions,
triangulate_planar_regions,
)
from .balloon_profile import balloon_profile as calculate_balloon_profile
from .mask_regions import alpha_regions
from .node_assets import cutout_node_group
from ..common.image_edit import (
ImageEditWarning,
polygon_bounds,
)
from ..common.image_mask import polygon_mask, visible_selection_mask
from .shape_types import (
BALLOON_ATTRIBUTE_NAME,
CUTOUT_SHAPE_INDICES,
)
CUTOUT_ALPHA_THRESHOLD = 0.1
CUTOUT_AXIS_TO_SOURCE = (
(0.0, -1.0, 0.0, 0.0),
(0.0, 0.0, 1.0, 0.0),
(-1.0, 0.0, 0.0, 0.0),
(0.0, 0.0, 0.0, 1.0),
)
def _surface_depth_image(source_object, points, valid, camera_depth):
import numpy as np
points = np.asarray(points, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
height, width = valid.shape
if points.shape != (height, width, 3):
raise ValueError("Depth Surface point dimensions do not match")
discontinuity = _depth_discontinuity_map(camera_depth, valid)
discontinuity[~valid] = -1.0
pixels = np.empty((height, width, 4), dtype=np.float32)
pixels[:, :, :3] = points
pixels[:, :, 3] = discontinuity
image = bpy.data.images.new(
depth_data_name(source_object),
width=width,
height=height,
alpha=True,
float_buffer=True,
)
image.colorspace_settings.name = "Non-Color"
image.alpha_mode = "CHANNEL_PACKED"
image.pixels.foreach_set(np.flipud(pixels).ravel())
image.pack()
return image
def _cutout_world_matrix(source_matrix, crop_center):
from mathutils import Matrix
center_x, center_y = (float(value) for value in crop_center)
return (
source_matrix
@ Matrix.Translation((center_x, center_y, 0.0))
@ Matrix(CUTOUT_AXIS_TO_SOURCE)
)
def _orient_cutout_vertices(vertices):
import numpy as np
vertices = np.asarray(vertices, dtype=np.float64)
return np.column_stack(
(
-vertices[:, 2],
-vertices[:, 0],
vertices[:, 1],
)
)
def _cutout_domain(
regions_pixels,
image_size,
bounds,
spacing,
world_metric,
balloon_algorithm="POISSON",
):
import numpy as np
x_min, x_max, y_min, y_max = bounds
regions, spacing = prepare_planar_regions(
regions_pixels,
image_size,
bounds,
spacing=spacing,
world_metric=world_metric,
)
planar, planar_faces, boundary, components = triangulate_planar_regions(
regions,
spacing,
return_components=True,
)
profile = calculate_balloon_profile(
balloon_algorithm, planar, planar_faces, boundary, components
)
faces = np.asarray(planar_faces, dtype=np.int32)
local = planar @ np.linalg.inv(metric_transform(world_metric))
uv = np.column_stack(
(
(local[:, 0] - x_min) / (x_max - x_min),
(local[:, 1] - y_min) / (y_max - y_min),
)
).astype(np.float32)
crop_center = np.asarray(
((x_min + x_max) * 0.5, (y_min + y_max) * 0.5),
dtype=np.float64,
)
local -= crop_center
vertices = np.column_stack((local, np.zeros(len(local), dtype=np.float64)))
return vertices, faces, uv, boundary, planar, profile
def _world_plane_metric(matrix_world):
from mathutils import Vector
linear = matrix_world.to_3x3()
x = linear @ Vector((1.0, 0.0, 0.0))
y = linear @ Vector((0.0, 1.0, 0.0))
z = linear @ Vector((0.0, 0.0, 1.0))
return ((x.dot(x), x.dot(y)), (x.dot(y), y.dot(y))), z.length
def _cutout_cleanup_area(spacing, world_metric):
import numpy as np
if spacing <= 0.0:
raise ValueError("Cutout point spacing must be greater than zero")
area_scale = abs(float(np.linalg.det(metric_transform(world_metric))))
if area_scale <= 1e-12:
raise ValueError("The Image Empty has a degenerate world transform")
return float(spacing) ** 2 / area_scale
def _load_field(path):
import numpy as np
with np.load(path, allow_pickle=False) as archive:
points = np.asarray(archive["points"], dtype=np.float32)
intrinsics = np.asarray(archive["intrinsics"], dtype=np.float32)
valid = np.asarray(archive["valid"], dtype=bool)
calibration_valid = np.asarray(archive["calibration_valid"], dtype=bool)
image_size = tuple(int(value) for value in archive["image_size"])
reference_depth = float(archive["reference_depth"])
if (
points.ndim != 3
or points.shape[2] != 3
or valid.shape != points.shape[:2]
or calibration_valid.shape != points.shape[:2]
):
raise ValueError("The mesh point field is invalid")
if intrinsics.shape != (3, 3) or not reference_depth > 0.0:
raise ValueError("The mesh camera field is invalid")
return points, intrinsics, valid, calibration_valid, image_size, reference_depth
def _surface_uniform_scale(
source_object,
field,
bounds=None,
):
import numpy as np
(
_points,
intrinsics,
_valid,
_calibration_valid,
image_size,
reference_depth,
) = field
width, height = image_size
x_min, x_max, y_min, y_max = bounds or plane_bounds(source_object)
plane_width = x_max - x_min
plane_height = y_max - y_min
fx, fy = float(intrinsics[0, 0]), float(intrinsics[1, 1])
if fx <= 0.0 or fy <= 0.0:
raise ValueError("Camera intrinsics contain an invalid focal length")
scale_x = plane_width * fx / (max(width - 1, 1) * reference_depth)
scale_y = plane_height * fy / (max(height - 1, 1) * reference_depth)
return float(np.sqrt(scale_x * scale_y))
def _depth_discontinuity_map(depth, valid):
import numpy as np
depth = np.asarray(depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool) & np.isfinite(depth) & (depth > 0.0)
padded_depth = np.pad(depth, 1, mode="edge")
padded_valid = np.pad(valid, 1, mode="edge")
windows = np.lib.stride_tricks.sliding_window_view(padded_depth, (3, 3))
usable = np.lib.stride_tricks.sliding_window_view(padded_valid, (3, 3))
minimum = np.min(np.where(usable, windows, np.inf), axis=(-2, -1))
maximum = np.max(np.where(usable, windows, -np.inf), axis=(-2, -1))
result = (maximum - minimum) / np.maximum(np.abs(depth), 1e-6)
result[~usable.all(axis=(-2, -1))] = 1.0
return result.astype(np.float32)
def _counter_clockwise_faces(points, faces):
import numpy as np
points = np.asarray(points, dtype=np.float64)
result = np.asarray(faces, dtype=np.int32).copy()
polygons = points[result]
following = np.roll(polygons, -1, axis=1)
signed_area = np.sum(
polygons[..., 0] * following[..., 1]
- polygons[..., 1] * following[..., 0],
axis=1,
)
result[signed_area < 0.0] = result[signed_area < 0.0, ::-1]
return result
def _assign_uv(mesh, uv):
import numpy as np
for layer in list(mesh.uv_layers):
mesh.uv_layers.remove(layer)
uv_layer = mesh.uv_layers.new(name="UVMap")
loop_vertices = np.empty(len(mesh.loops), dtype=np.int32)
mesh.loops.foreach_get("vertex_index", loop_vertices)
loop_uv = np.asarray(uv, dtype=np.float32)[loop_vertices]
uv_layer.data.foreach_set("uv", loop_uv.ravel())
def _assign_float_point_attribute(mesh, name, values):
import numpy as np
attribute = mesh.attributes.new(name=name, type="FLOAT", domain="POINT")
attribute.data.foreach_set("value", np.asarray(values, dtype=np.float32))
def _image_rgba(image):
import numpy as np
width, height = (int(value) for value in image.size)
pixels = np.empty(len(image.pixels), dtype=np.float32)
image.pixels.foreach_get(pixels)
return np.flipud(pixels.reshape((height, width, 4)))
def _mask_crop_bounds(alpha):
import numpy as np
values = np.asarray(alpha)
visible_y, visible_x = np.nonzero(values > CUTOUT_ALPHA_THRESHOLD)
if not len(visible_x):
raise ValueError("The Cutout selection contains no visible pixels")
return (
int(visible_x.min()),
int(visible_y.min()),
int(visible_x.max()) + 1,
int(visible_y.max()) + 1,
)
def _crop_plane_bounds(source_object, crop_bounds, source_size):
source_width, source_height = (float(value) for value in source_size)
left, top, right, bottom = (float(value) for value in crop_bounds)
x_min, x_max, y_min, y_max = plane_bounds(source_object)
width = x_max - x_min
height = y_max - y_min
return (
x_min + left * width / source_width,
x_min + right * width / source_width,
y_max - bottom * height / source_height,
y_max - top * height / source_height,
)
def _cropped_color_image(source_object, crop_bounds):
import numpy as np
source = source_object.data
rgba = _image_rgba(source)
left, top, right, bottom = crop_bounds
width = right - left
height = bottom - top
if width <= 0 or height <= 0:
raise ValueError("The Cutout crop dimensions must be positive")
cropped = np.zeros((height, width, 4), dtype=np.float32)
source_height, source_width = rgba.shape[:2]
source_left = max(0, left)
source_top = max(0, top)
source_right = min(source_width, right)
source_bottom = min(source_height, bottom)
if source_right > source_left and source_bottom > source_top:
cropped[
source_top - top : source_bottom - top,
source_left - left : source_right - left,
] = rgba[source_top:source_bottom, source_left:source_right]
image = bpy.data.images.new(
color_data_name(source_object),
width=width,
height=height,
alpha=True,
)
try:
image.colorspace_settings.name = source.colorspace_settings.name
image.alpha_mode = source.alpha_mode
image.pixels.foreach_set(np.flipud(cropped).ravel())
image.pack()
image.update()
except Exception:
bpy.data.images.remove(image, do_unlink=True)
raise
return image
def _cutout_regions(
alpha, mesh_detail, threshold=CUTOUT_ALPHA_THRESHOLD
):
contour_resolution = max(512, min(int(mesh_detail) * 8, 1024))
return alpha_regions(
alpha,
max_resolution=contour_resolution,
threshold=threshold,
)
def _antialias_selection(selection):
import numpy as np
values = np.asarray(selection, dtype=np.float32)
kernel = np.asarray((0.029, 0.235, 0.472, 0.235, 0.029), dtype=np.float32)
radius = len(kernel) // 2
horizontal = np.zeros_like(values)
padded = np.pad(values, ((0, 0), (radius, radius)), mode="edge")
for offset, weight in enumerate(kernel):
horizontal += padded[:, offset : offset + values.shape[1]] * weight
result = np.zeros_like(values)
padded = np.pad(horizontal, ((radius, radius), (0, 0)), mode="edge")
for offset, weight in enumerate(kernel):
result += padded[offset : offset + values.shape[0]] * weight
return result
def _geometry_cutout_regions(
alpha,
polygon,
mesh_detail,
invert=False,
refine_selection=False,
selection=None,
):
import numpy as np
values = np.asarray(alpha, dtype=np.float32)
if values.ndim != 2:
raise ValueError("Cutout Alpha must be two-dimensional")
source_height, source_width = values.shape
source_size = (source_width, source_height)
if refine_selection:
geometry_alpha = np.where(values > CUTOUT_ALPHA_THRESHOLD, values, 0.0)
regions, _image_size = _cutout_regions(geometry_alpha, mesh_detail)
return regions, source_size
geometry_alpha = selection
if geometry_alpha is None:
geometry_alpha = _antialias_selection(
polygon_mask(source_size, polygon, "NONZERO")
)
geometry_alpha = visible_selection_mask(
values,
geometry_alpha,
invert,
alpha_threshold=CUTOUT_ALPHA_THRESHOLD,
)
regions, _image_size = _cutout_regions(geometry_alpha, mesh_detail)
return regions, source_size
def _local_cutout_rgba(source_object, polygon, invert=False):
import numpy as np
rgba = _image_rgba(source_object.data).copy()
height, width = rgba.shape[:2]
selection = _antialias_selection(
polygon_mask((width, height), polygon, "NONZERO")
)
selection = visible_selection_mask(
rgba[:, :, 3],
selection,
invert,
alpha_threshold=CUTOUT_ALPHA_THRESHOLD,
)
rgba[:, :, 3] *= selection
rgba[rgba[:, :, 3] <= 0.0, :3] = 0.0
if not np.any(rgba[:, :, 3] > 0.0):
raise ImageEditWarning("The selection contains no visible image pixels")
return rgba, selection
def _new_material(
context,
source_object,
color_image,
normal_image=None,
normal_space="TANGENT",
):
return create_material(
material_data_name(source_object),
color_image,
0.0,
normal_image=normal_image,
normal_space=normal_space,
scene=context.scene,
)
def build_cutout_object(
context,
source_object,
shape,
detail,
material,
field=None,
regions=None,
image_size=None,
bounds=None,
keep_original=False,
refine_selection=False,
):
import numpy as np
bounds = bounds or plane_bounds(source_object)
crop_center = (
(bounds[0] + bounds[1]) * 0.5,
(bounds[2] + bounds[3]) * 0.5,
)
world_metric, world_z_scale = _world_plane_metric(source_object.matrix_world)
depth_image = None
depth_reference = None
depth_uniform_scale = None
depth_direction = None
preferences = addon_preferences(context)
balloon_algorithm = getattr(
preferences, "balloon_profile_algorithm", "POISSON"
)
point_spacing = _cutout_point_spacing(source_object, detail)
cleanup_area = _cutout_cleanup_area(point_spacing, world_metric)
domain = _cutout_domain(
regions,
image_size,
bounds,
point_spacing,
world_metric,
balloon_algorithm,
)
source_vertices, faces, uv, boundary, planar, balloon_profile = domain
faces = _counter_clockwise_faces(planar, faces)
vertices = _orient_cutout_vertices(source_vertices)
if world_z_scale <= 1e-10:
raise ValueError("The Image Empty has a degenerate world transform")
balloon_profile /= world_z_scale
if shape in {"DEPTH_BALLOON", "DEPTH_SURFACE"}:
(
points,
_intrinsics,
valid,
calibration_valid,
_image_size,
_reference,
) = field
depth_image = _surface_depth_image(
source_object, points, valid, points[..., 2]
)
depth_uniform_scale = _surface_uniform_scale(
source_object,
field,
bounds,
)
if not refine_selection:
depth_reference = UNREFINED_DEPTH_REFERENCE
if shape == "DEPTH_BALLOON":
depth_direction = DEFAULT_DEPTH_DIRECTION
elif shape == "DEPTH_SURFACE":
depth_reference = metric_depth_reference(
points[..., 2],
calibration_valid,
depth_uniform_scale,
)
else:
depth_reference, depth_direction = depth_balloon_projection_parameters(
vertices,
uv,
points[..., 2],
calibration_valid,
depth_uniform_scale,
)
source_name = source_object.name
mesh = bpy.data.meshes.new(source_name)
mesh.from_pydata(np.asarray(vertices).tolist(), [], faces)
for polygon in mesh.polygons:
polygon.use_smooth = True
_assign_uv(mesh, uv)
_assign_float_point_attribute(mesh, BALLOON_ATTRIBUTE_NAME, balloon_profile)
mesh.materials.append(material)
mesh.update()
mesh_object = bpy.data.objects.new(source_name, mesh)
context.collection.objects.link(mesh_object)
mesh_object.matrix_world = _cutout_world_matrix(
source_object.matrix_world, crop_center
)
mesh_object["anyimage_mesh_shape"] = shape
node_group = cutout_node_group(shape)
modifier = mesh_object.modifiers.new(name=node_group.name, type="NODES")
modifier.node_group = node_group
modifier.show_group_selector = False
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Shape"),
CUTOUT_SHAPE_INDICES[shape],
)
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Cleanup"),
cleanup_area,
)
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Smooth Iterations"),
0,
)
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Smooth"),
1.0,
)
thickness = 1.0 if shape in {"BALLOON", "DEPTH_BALLOON"} else 0.0
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Thickness"),
thickness,
)
if shape in {"DEPTH_BALLOON", "DEPTH_SURFACE"}:
set_modifier_input(
modifier,
modifier_input_identifier(
node_group,
"Depth Scale",
),
1.0,
)
if shape == "DEPTH_SURFACE":
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Uniform Scale"),
depth_uniform_scale,
)
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Reference Depth"),
depth_reference,
)
if shape == "DEPTH_BALLOON":
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Reference Depth"),
depth_reference,
)
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Depth Direction"),
depth_direction,
)
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Uniform Scale"),
depth_uniform_scale,
)
if depth_image is not None:
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Depth Image"),
depth_image,
)
for selected in context.selected_objects:
selected.select_set(False)
mesh_object.select_set(True)
context.view_layer.objects.active = mesh_object
if not keep_original:
bpy.data.objects.remove(source_object, do_unlink=True)
mesh_object.name = source_name
return mesh_object
def _load_normal_image(path, source_object):
image = bpy.data.images.load(str(path), check_existing=False)
image.name = normal_data_name(source_object)
image.colorspace_settings.name = "Non-Color"
image.pack()
return image
def _orient_cutout_object_normal(image):
import numpy as np
width, height = (int(value) for value in image.size)
pixels = np.empty(width * height * 4, dtype=np.float32)
image.pixels.foreach_get(pixels)
rgba = pixels.reshape((height, width, 4))
source = rgba[:, :, :3] * 2.0 - 1.0
oriented = np.empty_like(source)
oriented[:, :, 0] = -source[:, :, 2]
oriented[:, :, 1] = -source[:, :, 0]
oriented[:, :, 2] = source[:, :, 1]
rgba[:, :, :3] = np.clip(oriented * 0.5 + 0.5, 0.0, 1.0)
image.pixels.foreach_set(rgba.ravel())
image.update()
def limited_cutout_mesh_detail(mesh_detail, crop_size, max_input_size):
crop_width, crop_height = (int(value) for value in crop_size)
limited_width, limited_height = limited_image_size(
(crop_width, crop_height),
max_input_size,
)
scale = max(limited_width, limited_height) / max(crop_width, crop_height)
return max(4, min(1024, round(int(mesh_detail) * scale)))
def _cutout_point_spacing(source_object, detail):
import math
x_min, x_max, y_min, y_max = plane_bounds(source_object)
metric, _world_z_scale = _world_plane_metric(source_object.matrix_world)
width = (x_max - x_min) * math.sqrt(max(metric[0][0], 0.0))
height = (y_max - y_min) * math.sqrt(max(metric[1][1], 0.0))
return max(width, height) / max(int(detail), 8)
def build_local_cutout(
operator,
context,
source_object,
shape,
selection_data,
invert=False,
):
if video_total_frames(source_object.data) != 1:
raise ValueError("Mesh conversion currently supports still images")
polygon = json.loads(selection_data) if selection_data else None
if not isinstance(polygon, list) or len(polygon) < 3:
raise ValueError("The Cutout selection is invalid")
_rgba, selection = _local_cutout_rgba(source_object, polygon, invert)
source_size = tuple(source_object.data.size)
crop_bounds = _mask_crop_bounds(selection)
left, top, right, bottom = crop_bounds
cropped_selection = selection[top:bottom, left:right]
mesh_detail = limited_cutout_mesh_detail(
operator.mesh_detail,
(right - left, bottom - top),
configured_max_ai_input_size(context),
)
regions, image_size = _cutout_regions(
cropped_selection,
mesh_detail,
threshold=CUTOUT_ALPHA_THRESHOLD,
)
local_bounds = _crop_plane_bounds(
source_object,
crop_bounds,
source_size,
)
color_image = None
material = None
try:
color_image = _cropped_color_image(source_object, crop_bounds)
material = _new_material(context, source_object, color_image)
build_cutout_object(
context,
source_object,
shape,
mesh_detail,
material,
regions=regions,
image_size=image_size,
bounds=local_bounds,
keep_original=True,
)
except Exception:
if material is not None and material.users == 0:
bpy.data.materials.remove(material)
if color_image is not None and color_image.users == 0:
bpy.data.images.remove(color_image)
raise
push_undo(f"Convert to {shape.replace('_', ' ').title()}")
def build_cutout_response(context, result, operator):
source_object = bpy.data.objects.get(operator.source_object_name)
if not is_image_empty(source_object):
raise RuntimeError("The source Image Empty no longer exists")
source_size = tuple(source_object.data.size)
crop_bounds = tuple(
int(value)
for value in json.loads(
result.file("bounds").read_text(encoding="utf-8")
)
)
local_bounds = _crop_plane_bounds(
source_object,
crop_bounds,
source_size,
)
selection_image = bpy.data.images.load(
str(result.file("selection")),
check_existing=False,
)
try:
selection_alpha = _image_rgba(selection_image)[:, :, 3]
mesh_detail = limited_cutout_mesh_detail(
operator.mesh_detail,
selection_image.size,
operator.max_input_size,
)
regions, image_size = _cutout_regions(
selection_alpha,
mesh_detail,
threshold=CUTOUT_ALPHA_THRESHOLD,
)
finally:
bpy.data.images.remove(selection_image)
color_image = create_color_image(
source_object,
result.directory,
assign_to_source=False,
)
normal_path = result.directory / "normal.png"
normal_image = (
_load_normal_image(normal_path, source_object)
if normal_path.is_file()
else None
)
if normal_image is not None and operator.shape == "DEPTH_SURFACE":
_orient_cutout_object_normal(normal_image)
material = _new_material(
context,
source_object,
color_image,
normal_image,
normal_space=(
"OBJECT"
if operator.shape == "DEPTH_SURFACE" and normal_image is not None
else "TANGENT"
),
)
field = (
_load_field(result.file("field"))
if operator.shape in {"DEPTH_BALLOON", "DEPTH_SURFACE"}
else None
)
mesh_object = build_cutout_object(
context,
source_object,
operator.shape,
mesh_detail,
material,
field=field,
regions=regions,
image_size=image_size,
bounds=local_bounds,
keep_original=True,
refine_selection=operator.refine_selection,
)
if normal_image is not None and operator.shape != "DEPTH_SURFACE":
bake_tangent_normal_image(
context,
source_object,
mesh_object,
normal_image,
)
push_undo(f"Convert to {operator.shape.replace('_', ' ').title()}")

1
src/anyimage/operators/cutout_tool/geometry/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Cutout 的内存几何算法。"""

120
src/anyimage/operators/cutout_tool/balloon_profile.py → src/anyimage/operators/cutout_tool/geometry/balloon.py
View File

@@ -231,31 +231,49 @@ def cotangent_poisson_field(
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
faces = np.asarray(planar_faces, dtype=np.int64)
if faces.size == 0:
faces = np.empty((0, 3), dtype=np.int64)
if faces.ndim != 2 or faces.shape[1] != 3:
raise ValueError("Balloon profile faces must be triangles")
boundary_mask = np.asarray(boundary_mask, dtype=bool)
loads = np.zeros(len(points), dtype=np.float64)
edge_weights = {}
def add_edge(start, end, weight):
edge = tuple(sorted((int(start), int(end))))
edge_weights[edge] = edge_weights.get(edge, 0.0) + weight
for face in planar_faces:
first, second, third = (int(index) for index in face)
a, b, c = points[[first, second, third]]
cross = abs(
(b[0] - a[0]) * (c[1] - a[1])
- (b[1] - a[1]) * (c[0] - a[0])
triangles = points[faces]
a, b, c = (triangles[:, index] for index in range(3))
cross = np.abs(
(b[:, 0] - a[:, 0]) * (c[:, 1] - a[:, 1])
- (b[:, 1] - a[:, 1]) * (c[:, 0] - a[:, 0])
)
usable = cross > 1e-14
faces = faces[usable]
a, b, c = a[usable], b[usable], c[usable]
cross = cross[usable]
loads = np.bincount(
faces.ravel(),
weights=np.repeat(cross / 6.0, 3),
minlength=len(points),
)
edges = np.concatenate(
(
faces[:, (1, 2)],
faces[:, (0, 2)],
faces[:, (0, 1)],
),
axis=0,
)
edge_weights = np.concatenate(
(
np.sum((b - a) * (c - a), axis=1),
np.sum((a - b) * (c - b), axis=1),
np.sum((a - c) * (b - c), axis=1),
)
if cross <= 1e-14:
continue
area = cross * 0.5
loads[[first, second, third]] += area / 3.0
cotangent_a = float((b - a).dot(c - a) / cross)
cotangent_b = float((a - b).dot(c - b) / cross)
cotangent_c = float((a - c).dot(b - c) / cross)
add_edge(second, third, cotangent_a * 0.5)
add_edge(first, third, cotangent_b * 0.5)
add_edge(first, second, cotangent_c * 0.5)
) / np.tile(cross * 2.0, 3)
edges.sort(axis=1)
edges, inverse = np.unique(edges, axis=0, return_inverse=True)
edge_weights = np.bincount(
inverse,
weights=edge_weights,
minlength=len(edges),
)
interior = np.flatnonzero(~boundary_mask)
field = np.zeros(len(points), dtype=np.float64)
@@ -263,38 +281,46 @@ def cotangent_poisson_field(
return field
compact = np.full(len(points), -1, dtype=np.int64)
compact[interior] = np.arange(len(interior), dtype=np.int64)
diagonal = np.zeros(len(interior), dtype=np.float64)
edge_starts = []
edge_ends = []
weights = []
# Constrained triangulation can create negative cotangent weights.
# Clamping them keeps the Dirichlet system positive definite.
for (start, end), weight in edge_weights.items():
weight = max(float(weight), 0.0)
if weight <= 1e-14:
continue
start_compact = compact[start]
end_compact = compact[end]
if start_compact >= 0:
diagonal[start_compact] += weight
if end_compact >= 0:
diagonal[end_compact] += weight
if start_compact >= 0 and end_compact >= 0:
edge_starts.append(start_compact)
edge_ends.append(end_compact)
weights.append(weight)
edge_weights = np.maximum(edge_weights, 0.0)
usable = edge_weights > 1e-14
compact_edges = compact[edges[usable]]
edge_weights = edge_weights[usable]
start_compact = compact_edges[:, 0]
end_compact = compact_edges[:, 1]
start_usable = start_compact >= 0
end_usable = end_compact >= 0
diagonal = np.zeros(len(interior), dtype=np.float64)
diagonal += np.bincount(
start_compact[start_usable],
weights=edge_weights[start_usable],
minlength=len(interior),
)
diagonal += np.bincount(
end_compact[end_usable],
weights=edge_weights[end_usable],
minlength=len(interior),
)
if np.any(diagonal <= 1e-14):
raise ValueError("Unable to build a stable Balloon profile")
edge_starts = np.asarray(edge_starts, dtype=np.int64)
edge_ends = np.asarray(edge_ends, dtype=np.int64)
weights = np.asarray(weights, dtype=np.float64)
interior_edges = start_usable & end_usable
edge_starts = start_compact[interior_edges]
edge_ends = end_compact[interior_edges]
weights = edge_weights[interior_edges]
directed_starts = np.concatenate((edge_starts, edge_ends))
directed_ends = np.concatenate((edge_ends, edge_starts))
directed_weights = np.concatenate((weights, weights))
def multiply(values):
result = diagonal * values
if len(weights):
np.add.at(result, edge_starts, -weights * values[edge_ends])
np.add.at(result, edge_ends, -weights * values[edge_starts])
if len(directed_weights):
result -= np.bincount(
directed_starts,
weights=directed_weights * values[directed_ends],
minlength=len(values),
)
return result
right_hand_side = loads[interior]

11
src/anyimage/operators/cutout_tool/depth_calibration.py → src/anyimage/operators/cutout_tool/geometry/depth_calibration.py
View File

@@ -3,10 +3,19 @@
PLANE_VARIANCE_FLOOR_RATIO = 0.025
DEPTH_REFERENCE_PERCENTILE = 95.0
UNREFINED_DEPTH_REFERENCE = 8.0
ADAPTIVE_DEPTH_TRANSPARENCY_RATIO = 0.05
DEFAULT_DEPTH_BASELINE = 8.0
DEFAULT_DEPTH_DIRECTION = (0.0, 0.0, 1.0)
def use_adaptive_depth_calibration(region_transparent_fraction):
"""Return whether the Region input has enough transparent context."""
return (
float(region_transparent_fraction)
>= ADAPTIVE_DEPTH_TRANSPARENCY_RATIO
)
def metric_depth_reference(camera_depth, valid, uniform_scale):
"""Return the shared upper-percentile scene depth for both Depth shapes."""
import numpy as np

81
src/anyimage/operators/cutout_tool/mask_regions.py → src/anyimage/operators/cutout_tool/geometry/region_contours.py
View File

@@ -1,4 +1,34 @@
"""Convert a Cutout Alpha mask into outer contours and holes."""
"""Extract Cutout Region contours from a Region Mask."""
from dataclasses import dataclass
@dataclass(frozen=True)
class RegionContours:
components: tuple
image_size: tuple[int, int]
def __post_init__(self):
components = tuple(
tuple(
tuple((float(x), float(y)) for x, y in contour)
for contour in component
)
for component in self.components
)
image_size = tuple(int(value) for value in self.image_size)
if not components or any(not component for component in components):
raise ValueError("The Region has no usable contour components")
if any(
len(contour) < 3
for component in components
for contour in component
):
raise ValueError("A Region contour has fewer than three points")
if len(image_size) != 2 or min(image_size) <= 0:
raise ValueError("The Region contour image size is invalid")
object.__setattr__(self, "components", components)
object.__setattr__(self, "image_size", image_size)
def _choose_boundary_edge(start, direction, candidates):
@@ -27,7 +57,7 @@ def _simplify_axis_aligned_contour(points):
return simplified
def trace_mask_contours(mask):
def _trace_region_contours(mask):
import numpy as np
mask = np.asarray(mask, dtype=bool)
@@ -113,8 +143,8 @@ def _point_inside_contour(point, contour):
return inside
def mask_regions(mask):
contours = trace_mask_contours(mask)
def _group_region_contours(mask):
contours = _trace_region_contours(mask)
outer = [contour for contour in contours if _signed_contour_area(contour) > 0.0]
holes = [contour for contour in contours if _signed_contour_area(contour) < 0.0]
if not outer:
@@ -143,7 +173,7 @@ def mask_regions(mask):
return regions
def _resize_alpha(values, size):
def _resize_values(values, size):
import numpy as np
target_width, target_height = (int(value) for value in size)
@@ -174,10 +204,10 @@ def _resize_alpha(values, size):
)
def alpha_regions(alpha, max_resolution=512, threshold=0.5):
def extract_region_contours(region_mask, max_resolution=512, threshold=0.5):
import numpy as np
values = np.asarray(alpha)
values = np.asarray(region_mask.values)
if values.ndim != 2:
raise ValueError("Cutout Alpha must be two-dimensional")
if np.issubdtype(values.dtype, np.floating):
@@ -188,33 +218,24 @@ def alpha_regions(alpha, max_resolution=512, threshold=0.5):
scale = min(1.0, float(max_resolution) / max(source_size))
sampled_size = tuple(max(1, round(value * scale)) for value in source_size)
if sampled_size != source_size:
values = _resize_alpha(values, sampled_size)
values = _resize_values(values, sampled_size)
mask = values >= 255.0 * threshold
regions = mask_regions(mask)
components = _group_region_contours(mask)
scale_x = source_size[0] / sampled_size[0]
scale_y = source_size[1] / sampled_size[1]
return (
[
[
[
[
return RegionContours(
components=tuple(
tuple(
tuple(
(
min(max(float(x) * scale_x, 0.0), source_size[0]),
min(max(float(y) * scale_y, 0.0), source_size[1]),
]
)
for x, y in contour
]
for contour in region
]
for region in regions
],
source_size,
)
for contour in component
)
for component in components
),
image_size=source_size,
)
def image_alpha_regions(path, max_resolution=512, threshold=0.5):
import numpy as np
from PIL import Image
with Image.open(path) as source:
alpha = np.asarray(source.convert("RGBA").getchannel("A"), dtype=np.uint8)
return alpha_regions(alpha, max_resolution=max_resolution, threshold=threshold)

94
src/anyimage/operators/cutout_tool/planar_mesh.py → src/anyimage/operators/cutout_tool/geometry/triangulation.py
View File

@@ -3,7 +3,7 @@ import math
POINT_SPACING = 0.1
def polygon_area(points):
def contour_area(points):
import numpy as np
points = np.asarray(points, dtype=np.float64)
@@ -16,10 +16,10 @@ def polygon_area(points):
)
def resample_closed_polygon(polygon, spacing):
def resample_closed_contour(contour, spacing):
import numpy as np
points = np.asarray(polygon, dtype=np.float64)
points = np.asarray(contour, dtype=np.float64)
if len(points) < 3:
raise ValueError("The lasso does not define a valid image region")
segments = np.roll(points, -1, axis=0) - points
@@ -40,9 +40,9 @@ def resample_closed_polygon(polygon, spacing):
return points[indices] + segments[indices] * factors[:, None]
def smooth_lasso_contour(polygon, spacing, iterations=16):
contour = resample_closed_polygon(polygon, spacing * 0.25)
original_area = abs(polygon_area(contour))
def smooth_region_contour(contour, spacing, iterations=16):
contour = resample_closed_contour(contour, spacing * 0.25)
original_area = abs(contour_area(contour))
for _index in range(iterations):
import numpy as np
@@ -51,14 +51,14 @@ def smooth_lasso_contour(polygon, spacing, iterations=16):
+ contour * 0.75
+ np.roll(contour, -1, axis=0) * 0.125
)
smooth_area = abs(polygon_area(contour))
smooth_area = abs(contour_area(contour))
if original_area > 0.0 and smooth_area > 0.0:
center = contour.mean(axis=0)
contour = center + (contour - center) * math.sqrt(
original_area / smooth_area
)
contour = resample_closed_polygon(contour, spacing)
if polygon_area(contour) < 0.0:
contour = resample_closed_contour(contour, spacing)
if contour_area(contour) < 0.0:
contour = contour[::-1].copy()
return contour
@@ -98,10 +98,6 @@ def _edge_bucket_indices(starts, ends, minimum_y, bucket_height, bucket_count):
return buckets
def points_inside_polygon(points, polygon):
return points_inside_contours(points, (polygon,))
def points_inside_contours(points, contours):
import numpy as np
@@ -236,10 +232,6 @@ def uniform_interior_points_for_contours(contours, spacing):
)
def uniform_interior_points(contour, spacing):
return uniform_interior_points_for_contours((contour,), spacing)
def metric_transform(world_metric=None):
import numpy as np
@@ -259,7 +251,7 @@ def metric_transform(world_metric=None):
def prepare_planar_contours(
polygons,
contours,
image_size,
local_bounds,
spacing=POINT_SPACING,
@@ -274,9 +266,9 @@ def prepare_planar_contours(
local_width = x_max - x_min
local_height = y_max - y_min
transform = metric_transform(world_metric)
contours = []
for polygon in polygons:
pixels = np.asarray(polygon, dtype=np.float64)
prepared = []
for contour in contours:
pixels = np.asarray(contour, dtype=np.float64)
if pixels.ndim != 2 or pixels.shape[0] < 3 or pixels.shape[1] != 2:
continue
local = np.empty_like(pixels)
@@ -289,30 +281,29 @@ def prepare_planar_contours(
if perimeter <= 1e-12:
continue
contour_spacing = min(spacing, perimeter / 8.0)
smoothed = smooth_lasso_contour(planar, contour_spacing)
smoothed = smooth_region_contour(planar, contour_spacing)
smoothed_local = smoothed @ np.linalg.inv(transform)
smoothed_local[:, 0] = np.clip(smoothed_local[:, 0], x_min, x_max)
smoothed_local[:, 1] = np.clip(smoothed_local[:, 1], y_min, y_max)
contours.append(smoothed_local @ transform)
if not contours:
prepared.append(smoothed_local @ transform)
if not prepared:
raise ValueError("The Cutout has no region large enough for this resolution")
interior = uniform_interior_points_for_contours(contours, spacing)
return contours, interior, spacing
interior = uniform_interior_points_for_contours(prepared, spacing)
return prepared, interior, spacing
def prepare_planar_regions(
regions,
image_size,
def _prepare_region_contours(
region_contours,
local_bounds,
spacing=POINT_SPACING,
world_metric=None,
):
prepared = []
errors = []
for region in regions:
for component in region_contours.components:
try:
contours, interior, _spacing = prepare_planar_contours(
region,
image_size,
component,
region_contours.image_size,
local_bounds,
spacing=spacing,
world_metric=world_metric,
@@ -328,25 +319,7 @@ def prepare_planar_regions(
return prepared, spacing
def prepare_planar_points(
polygon,
image_size,
local_bounds,
spacing=POINT_SPACING,
world_metric=None,
):
contours, interior, spacing = prepare_planar_contours(
(polygon,),
image_size,
local_bounds,
spacing=spacing,
world_metric=world_metric,
)
contour = contours[0]
return contour, interior, spacing
def triangulate_planar_contours(contours, interior, spacing):
def _triangulate_contour_component(contours, interior, spacing):
import numpy as np
from mathutils import Vector
from mathutils.geometry import delaunay_2d_cdt
@@ -413,16 +386,28 @@ def boundary_vertices_from_faces(vertex_count, faces):
return boundary
def triangulate_planar_regions(regions, spacing, return_components=False):
def triangulate_region_contours(
region_contours,
local_bounds,
spacing=POINT_SPACING,
world_metric=None,
return_components=False,
):
import numpy as np
prepared, spacing = _prepare_region_contours(
region_contours,
local_bounds,
spacing=spacing,
world_metric=world_metric,
)
points = []
faces = []
boundaries = []
components = []
offset = 0
for contours, interior in regions:
region_points, region_faces, region_boundary = triangulate_planar_contours(
for contours, interior in prepared:
region_points, region_faces, region_boundary = _triangulate_contour_component(
contours,
interior,
spacing,
@@ -449,4 +434,3 @@ def triangulate_planar_regions(regions, spacing, return_components=False):
np.concatenate(boundaries),
)
return (*result, components) if return_components else result

10
src/anyimage/operators/cutout_tool/node_assets.py
View File

@@ -1,10 +0,0 @@
"""Load Geometry Nodes assets used by Cutout shapes."""
from ..common.node_asset import load_asset_node_group
from .shape_types import CUTOUT_NODE_GROUP_NAME, CUTOUT_SHAPES
def cutout_node_group(shape):
if shape not in CUTOUT_SHAPES:
raise ValueError(f"Unsupported Cutout shape: {shape}")
return load_asset_node_group(CUTOUT_NODE_GROUP_NAME)

842
src/anyimage/operators/cutout_tool/region_to_shape.py Normal file
View File

@@ -0,0 +1,842 @@
import json
from dataclasses import dataclass
from enum import StrEnum
import bpy
from ...preferences import addon_preferences, configured_max_ai_input_size
from ...properties import BEN2_MODEL_KEY, DEFAULT_MOGE2_MODEL_KEY
from ...runtime import JobOperatorBase
from ...server.media.resolution import limited_image_size
from ...server.media.resolution import (
DEFAULT_MAX_AI_INPUT_SIZE,
MAX_MAX_AI_INPUT_SIZE,
MIN_MAX_AI_INPUT_SIZE,
)
from ..common.image_data import (
cleanup_image_input,
color_data_name,
depth_data_name,
image_rgba,
load_normal_result_image,
image_empty_bounds,
prepare_region_input,
require_image_empty,
source_start_frame,
video_total_frames,
)
from ..common.image_interaction import report_image_edit_exception
from ..common.image_region import (
ImageEditWarning,
RegionMask,
RegionPath,
rasterize_region_path,
)
from ..common.mesh_result import (
create_image_material,
finalize_object_result,
load_asset_node_group,
load_image_geometry,
modifier_input_identifier,
set_modifier_input,
)
from ..common.ai_job import (
invoke_ai_setup_if_needed,
moge2_parameters,
production_device,
require_environment,
require_input_path,
require_model,
)
from .geometry.depth_calibration import (
DEFAULT_DEPTH_BASELINE,
DEFAULT_DEPTH_DIRECTION,
depth_balloon_projection_parameters,
metric_depth_reference,
use_adaptive_depth_calibration,
)
from .geometry.triangulation import (
metric_transform,
triangulate_region_contours,
)
from .geometry.balloon import balloon_profile as calculate_balloon_profile
from .geometry.region_contours import extract_region_contours
CUTOUT_ALPHA_THRESHOLD = 0.1
CUTOUT_CLEANUP_UI_SCALE = 100.0
BALLOON_ATTRIBUTE_NAME = "o_balloon"
CUTOUT_NODE_GROUP_NAME = "O Image Cutout"
CUTOUT_NODE_GROUP_NAMES = {
"SURFACE": "O Image Surface",
"BALLOON": "O Image Balloon",
"DEPTH_BALLOON": "O Image Depth Balloon",
"DEPTH_SURFACE": "O Image Depth Surface",
}
CUTOUT_SHAPE_INDICES = {
shape: index for index, shape in enumerate(CUTOUT_NODE_GROUP_NAMES)
}
CUTOUT_SHAPES = frozenset(CUTOUT_NODE_GROUP_NAMES)
DEPTH_CUTOUT_SHAPES = frozenset({"DEPTH_BALLOON", "DEPTH_SURFACE"})
_HIDDEN = {"HIDDEN", "SKIP_SAVE"}
class NormalMode(StrEnum):
NONE = "NONE"
TANGENT = "TANGENT"
OBJECT = "OBJECT"
@dataclass(frozen=True)
class BaseShape:
vertices: object
faces: object
uv: object
balloon: object
CUTOUT_AXIS_TO_SOURCE = (
(0.0, -1.0, 0.0, 0.0),
(0.0, 0.0, 1.0, 0.0),
(-1.0, 0.0, 0.0, 0.0),
(0.0, 0.0, 0.0, 1.0),
)
def _cutout_depth_image(source_object, points, valid, camera_depth):
import numpy as np
points = np.asarray(points, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
height, width = valid.shape
if points.shape != (height, width, 3):
raise ValueError("Depth Surface point dimensions do not match")
discontinuity = _depth_discontinuity_map(camera_depth, valid)
discontinuity[~valid] = -1.0
pixels = np.empty((height, width, 4), dtype=np.float32)
pixels[:, :, :3] = points
pixels[:, :, 3] = discontinuity
image = bpy.data.images.new(
depth_data_name(source_object),
width=width,
height=height,
alpha=True,
float_buffer=True,
)
image.colorspace_settings.name = "Non-Color"
image.alpha_mode = "CHANNEL_PACKED"
image.pixels.foreach_set(np.flipud(pixels).ravel())
image.pack()
return image
def _cutout_world_matrix(source_matrix, crop_center):
from mathutils import Matrix
center_x, center_y = (float(value) for value in crop_center)
return (
source_matrix
@ Matrix.Translation((center_x, center_y, 0.0))
@ Matrix(CUTOUT_AXIS_TO_SOURCE)
)
def _orient_cutout_vertices(vertices):
import numpy as np
vertices = np.asarray(vertices, dtype=np.float64)
return np.column_stack(
(
-vertices[:, 2],
-vertices[:, 0],
vertices[:, 1],
)
)
def _cutout_domain(
region_contours,
bounds,
spacing,
world_metric,
balloon_algorithm="POISSON",
):
import numpy as np
x_min, x_max, y_min, y_max = bounds
planar, planar_faces, boundary, components = triangulate_region_contours(
region_contours,
bounds,
spacing=spacing,
world_metric=world_metric,
return_components=True,
)
profile = calculate_balloon_profile(
balloon_algorithm, planar, planar_faces, boundary, components
)
faces = np.asarray(planar_faces, dtype=np.int32)
local = planar @ np.linalg.inv(metric_transform(world_metric))
uv = np.column_stack(
(
(local[:, 0] - x_min) / (x_max - x_min),
(local[:, 1] - y_min) / (y_max - y_min),
)
).astype(np.float32)
crop_center = np.asarray(
((x_min + x_max) * 0.5, (y_min + y_max) * 0.5),
dtype=np.float64,
)
local -= crop_center
vertices = np.column_stack((local, np.zeros(len(local), dtype=np.float64)))
return vertices, faces, uv, boundary, planar, profile
def _world_plane_metric(matrix_world):
from mathutils import Vector
linear = matrix_world.to_3x3()
x = linear @ Vector((1.0, 0.0, 0.0))
y = linear @ Vector((0.0, 1.0, 0.0))
z = linear @ Vector((0.0, 0.0, 1.0))
return ((x.dot(x), x.dot(y)), (x.dot(y), y.dot(y))), z.length
def _cutout_cleanup_value(spacing, world_metric):
import numpy as np
if spacing <= 0.0:
raise ValueError("Cutout point spacing must be greater than zero")
area_scale = abs(float(np.linalg.det(metric_transform(world_metric))))
if area_scale <= 1e-12:
raise ValueError("The Image Empty has a degenerate world transform")
return CUTOUT_CLEANUP_UI_SCALE * float(spacing) ** 2 / area_scale
def _depth_uniform_scale(
source_object,
field,
bounds=None,
):
import numpy as np
intrinsics = field.intrinsics
width, height = field.image_size
x_min, x_max, y_min, y_max = bounds or image_empty_bounds(source_object)
plane_width = x_max - x_min
plane_height = y_max - y_min
fx, fy = float(intrinsics[0, 0]), float(intrinsics[1, 1])
if fx <= 0.0 or fy <= 0.0:
raise ValueError("Camera intrinsics contain an invalid focal length")
scale_x = plane_width * fx / (max(width - 1, 1) * field.reference_depth)
scale_y = plane_height * fy / (max(height - 1, 1) * field.reference_depth)
return float(np.sqrt(scale_x * scale_y))
def _depth_discontinuity_map(depth, valid):
import numpy as np
depth = np.asarray(depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool) & np.isfinite(depth) & (depth > 0.0)
padded_depth = np.pad(depth, 1, mode="edge")
padded_valid = np.pad(valid, 1, mode="edge")
windows = np.lib.stride_tricks.sliding_window_view(padded_depth, (3, 3))
usable = np.lib.stride_tricks.sliding_window_view(padded_valid, (3, 3))
minimum = np.min(np.where(usable, windows, np.inf), axis=(-2, -1))
maximum = np.max(np.where(usable, windows, -np.inf), axis=(-2, -1))
result = (maximum - minimum) / np.maximum(np.abs(depth), 1e-6)
result[~usable.all(axis=(-2, -1))] = 1.0
return result.astype(np.float32)
def _counter_clockwise_faces(points, faces):
import numpy as np
points = np.asarray(points, dtype=np.float64)
result = np.asarray(faces, dtype=np.int32).copy()
polygons = points[result]
following = np.roll(polygons, -1, axis=1)
signed_area = np.sum(
polygons[..., 0] * following[..., 1]
- polygons[..., 1] * following[..., 0],
axis=1,
)
result[signed_area < 0.0] = result[signed_area < 0.0, ::-1]
return result
def _assign_uv(mesh, uv):
import numpy as np
for layer in list(mesh.uv_layers):
mesh.uv_layers.remove(layer)
uv_layer = mesh.uv_layers.new(name="UVMap")
loop_vertices = np.empty(len(mesh.loops), dtype=np.int32)
mesh.loops.foreach_get("vertex_index", loop_vertices)
loop_uv = np.asarray(uv, dtype=np.float32)[loop_vertices]
uv_layer.data.foreach_set("uv", loop_uv.ravel())
def _assign_float_point_attribute(mesh, name, values):
import numpy as np
attribute = mesh.attributes.new(name=name, type="FLOAT", domain="POINT")
attribute.data.foreach_set("value", np.asarray(values, dtype=np.float32))
def _crop_plane_bounds(source_object, crop_bounds, source_size):
source_width, source_height = (float(value) for value in source_size)
left, top, right, bottom = (float(value) for value in crop_bounds)
x_min, x_max, y_min, y_max = image_empty_bounds(source_object)
width = x_max - x_min
height = y_max - y_min
return (
x_min + left * width / source_width,
x_min + right * width / source_width,
y_max - bottom * height / source_height,
y_max - top * height / source_height,
)
def _cropped_color_image(source_object, crop_bounds, source_rgba):
import numpy as np
source = source_object.data
rgba = np.asarray(source_rgba)
left, top, right, bottom = crop_bounds
width = right - left
height = bottom - top
if width <= 0 or height <= 0:
raise ValueError("The Cutout crop dimensions must be positive")
cropped = np.zeros((height, width, 4), dtype=np.float32)
source_height, source_width = rgba.shape[:2]
source_left = max(0, left)
source_top = max(0, top)
source_right = min(source_width, right)
source_bottom = min(source_height, bottom)
if source_right > source_left and source_bottom > source_top:
cropped[
source_top - top : source_bottom - top,
source_left - left : source_right - left,
] = rgba[source_top:source_bottom, source_left:source_right]
image = bpy.data.images.new(
color_data_name(source_object),
width=width,
height=height,
alpha=True,
)
try:
image.colorspace_settings.name = source.colorspace_settings.name
image.alpha_mode = source.alpha_mode
image.pixels.foreach_set(np.flipud(cropped).ravel())
image.pack()
image.update()
except Exception:
bpy.data.images.remove(image, do_unlink=True)
raise
return image
def _extract_cutout_contours(
region_mask,
mesh_detail,
threshold=CUTOUT_ALPHA_THRESHOLD,
):
contour_resolution = max(512, min(int(mesh_detail) * 8, 1024))
return extract_region_contours(
region_mask,
max_resolution=contour_resolution,
threshold=threshold,
)
def cutout_node_group(shape):
if shape not in CUTOUT_SHAPES:
raise ValueError(f"Unsupported Cutout shape: {shape}")
return load_asset_node_group(CUTOUT_NODE_GROUP_NAME)
def limited_cutout_mesh_detail(mesh_detail, crop_size, max_input_size):
crop_width, crop_height = (int(value) for value in crop_size)
limited_width, limited_height = limited_image_size(
(crop_width, crop_height),
max_input_size,
)
scale = max(limited_width, limited_height) / max(crop_width, crop_height)
return max(4, min(1024, round(int(mesh_detail) * scale)))
def _cutout_point_spacing(source_object, detail):
import math
x_min, x_max, y_min, y_max = image_empty_bounds(source_object)
metric, _world_z_scale = _world_plane_metric(source_object.matrix_world)
width = (x_max - x_min) * math.sqrt(max(metric[0][0], 0.0))
height = (y_max - y_min) * math.sqrt(max(metric[1][1], 0.0))
return max(width, height) / max(int(detail), 8)
def build_base_shape(context, source_object, detail, region_mask):
import numpy as np
bounds = _crop_plane_bounds(
source_object,
region_mask.bounds,
tuple(source_object.data.size),
)
region_contours = _extract_cutout_contours(
region_mask,
detail,
threshold=CUTOUT_ALPHA_THRESHOLD,
)
world_metric, world_z_scale = _world_plane_metric(source_object.matrix_world)
if world_z_scale <= 1e-10:
raise ValueError("The Image Empty has a degenerate world transform")
preferences = addon_preferences(context)
balloon_algorithm = getattr(
preferences, "balloon_profile_algorithm", "POISSON"
)
point_spacing = _cutout_point_spacing(source_object, detail)
domain = _cutout_domain(
region_contours,
bounds,
point_spacing,
world_metric,
balloon_algorithm,
)
source_vertices, faces, uv, _boundary, planar, balloon = domain
return BaseShape(
vertices=_orient_cutout_vertices(source_vertices),
faces=_counter_clockwise_faces(planar, faces),
uv=np.asarray(uv, dtype=np.float32),
balloon=np.asarray(balloon, dtype=np.float64) / world_z_scale,
)
def create_shape_object(
context,
source_object,
shape,
detail,
region_mask,
base_shape,
*,
field=None,
normal_image=None,
normal_mode=NormalMode.NONE,
source_rgba=None,
):
import numpy as np
normal_mode = NormalMode(normal_mode)
bounds = _crop_plane_bounds(
source_object,
region_mask.bounds,
tuple(source_object.data.size),
)
crop_center = (
(bounds[0] + bounds[1]) * 0.5,
(bounds[2] + bounds[3]) * 0.5,
)
world_metric, _world_z_scale = _world_plane_metric(source_object.matrix_world)
cleanup_value = _cutout_cleanup_value(
_cutout_point_spacing(source_object, detail),
world_metric,
)
color_image = _cropped_color_image(
source_object,
region_mask.bounds,
image_rgba(source_object.data) if source_rgba is None else source_rgba,
)
material = None
depth_image = None
mesh = None
mesh_object = None
try:
material = create_image_material(
source_object,
color_image,
0.0,
normal_image=normal_image,
normal_space=(
NormalMode.TANGENT.value
if normal_mode == NormalMode.NONE
else normal_mode.value
),
scene=context.scene,
)
depth_reference = None
depth_uniform_scale = None
depth_direction = None
if shape in DEPTH_CUTOUT_SHAPES:
points = field.points
valid = field.valid
calibration_valid = field.calibration_valid
depth_image = _cutout_depth_image(
source_object, points, valid, points[..., 2]
)
depth_uniform_scale = _depth_uniform_scale(
source_object,
field,
bounds,
)
if not use_adaptive_depth_calibration(
field.region_transparent_fraction
):
depth_reference = DEFAULT_DEPTH_BASELINE
if shape == "DEPTH_BALLOON":
depth_direction = DEFAULT_DEPTH_DIRECTION
elif shape == "DEPTH_SURFACE":
depth_reference = metric_depth_reference(
points[..., 2],
calibration_valid,
depth_uniform_scale,
)
else:
depth_reference, depth_direction = (
depth_balloon_projection_parameters(
base_shape.vertices,
base_shape.uv,
points[..., 2],
calibration_valid,
depth_uniform_scale,
)
)
source_name = source_object.name
mesh = bpy.data.meshes.new(source_name)
mesh.from_pydata(
np.asarray(base_shape.vertices).tolist(),
[],
base_shape.faces,
)
for face in mesh.polygons:
face.use_smooth = True
_assign_uv(mesh, base_shape.uv)
_assign_float_point_attribute(
mesh,
BALLOON_ATTRIBUTE_NAME,
base_shape.balloon,
)
mesh.materials.append(material)
mesh.update()
mesh_object = bpy.data.objects.new(source_name, mesh)
context.collection.objects.link(mesh_object)
mesh_object.matrix_world = _cutout_world_matrix(
source_object.matrix_world,
crop_center,
)
mesh_object["anyimage_mesh_shape"] = shape
node_group = cutout_node_group(shape)
modifier = mesh_object.modifiers.new(name=node_group.name, type="NODES")
modifier.node_group = node_group
modifier.show_group_selector = False
for name, value in (
("Shape", CUTOUT_SHAPE_INDICES[shape]),
("Cleanup", cleanup_value),
("Smooth Iterations", 0),
("Smooth", 1.0),
(
"Thickness",
1.0 if shape in {"BALLOON", "DEPTH_BALLOON"} else 0.0,
),
):
set_modifier_input(
modifier,
modifier_input_identifier(node_group, name),
value,
)
if shape in DEPTH_CUTOUT_SHAPES:
for name, value in (
("Depth Scale", 1.0),
("Uniform Scale", depth_uniform_scale),
("Base Plane Depth", depth_reference),
("Depth Image", depth_image),
):
set_modifier_input(
modifier,
modifier_input_identifier(node_group, name),
value,
)
if shape == "DEPTH_BALLOON":
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Depth Direction"),
depth_direction,
)
return finalize_object_result(
context,
source_object,
mesh_object,
keep_source=True,
)
except Exception:
if mesh_object is not None:
bpy.data.objects.remove(mesh_object, do_unlink=True)
if mesh is not None and mesh.users == 0:
bpy.data.meshes.remove(mesh)
if material is not None and material.users == 0:
bpy.data.materials.remove(material)
if color_image.users == 0:
bpy.data.images.remove(color_image, do_unlink=True)
if normal_image is not None and normal_image.users == 0:
bpy.data.images.remove(normal_image, do_unlink=True)
if depth_image is not None and depth_image.users == 0:
bpy.data.images.remove(depth_image, do_unlink=True)
raise
def region_to_shape(
context,
source_object,
shape,
detail,
region_mask,
*,
field=None,
normal_image=None,
normal_mode=NormalMode.NONE,
source_rgba=None,
):
crop_width = region_mask.bounds[2] - region_mask.bounds[0]
crop_height = region_mask.bounds[3] - region_mask.bounds[1]
detail = limited_cutout_mesh_detail(
detail,
(crop_width, crop_height),
configured_max_ai_input_size(context),
)
base_shape = build_base_shape(
context,
source_object,
detail,
region_mask,
)
result = create_shape_object(
context,
source_object,
shape,
detail,
region_mask,
base_shape,
field=field,
normal_image=normal_image,
normal_mode=normal_mode,
source_rgba=source_rgba,
)
undo_message = f"Convert to {shape.replace('_', ' ').title()}"
if "FINISHED" not in bpy.ops.ed.undo_push(message=undo_message):
raise RuntimeError(f"Unable to create the {undo_message} Undo step")
return result
def _region_mask_from_image(path, bounds):
image = bpy.data.images.load(str(path), check_existing=False)
try:
return RegionMask(
image_rgba(image)[:, :, 3],
tuple(int(value) for value in bounds),
)
finally:
bpy.data.images.remove(image, do_unlink=True)
def _generated_region_mask(result, input_path, source_bounds):
if not isinstance(result.value, dict) or "region" not in result.value:
return _region_mask_from_image(input_path, source_bounds)
local_bounds = tuple(
int(value)
for value in json.loads(
result.file("region_bounds").read_text(encoding="utf-8")
)
)
bounds = (
source_bounds[0] + local_bounds[0],
source_bounds[1] + local_bounds[1],
source_bounds[0] + local_bounds[2],
source_bounds[1] + local_bounds[3],
)
return _region_mask_from_image(result.file("region"), bounds)
def cutout_moge_settings(context):
preferences = addon_preferences(context)
model = getattr(preferences, "mesh_depth_model", DEFAULT_MOGE2_MODEL_KEY)
if not model.startswith("MOGE2_"):
model = DEFAULT_MOGE2_MODEL_KEY
return (
model,
int(getattr(preferences, "mesh_resolution_level", 5)),
)
class CutoutRegionToShape(JobOperatorBase, bpy.types.Operator):
bl_idname = "anyimage.cutout_region_to_shape"
bl_label = "Cutout Region to Shape"
bl_description = "Build the selected image Region as a Cutout shape"
bl_options = {"INTERNAL"}
job_type = "generate-region-data"
starting_message = "Generating Cutout Region data"
input_path: bpy.props.StringProperty(options=_HIDDEN)
source_object_name: bpy.props.StringProperty(options=_HIDDEN)
region_path_json: bpy.props.StringProperty(options=_HIDDEN)
region_bounds_json: bpy.props.StringProperty(options=_HIDDEN)
shape: bpy.props.StringProperty(options=_HIDDEN)
mesh_detail: bpy.props.IntProperty(
default=128,
min=4,
max=1024,
options=_HIDDEN,
)
model: bpy.props.StringProperty(
default=DEFAULT_MOGE2_MODEL_KEY,
options=_HIDDEN,
)
resolution_level: bpy.props.IntProperty(
default=5,
min=0,
max=9,
options=_HIDDEN,
)
max_input_size: bpy.props.IntProperty(
default=DEFAULT_MAX_AI_INPUT_SIZE,
min=MIN_MAX_AI_INPUT_SIZE,
max=MAX_MAX_AI_INPUT_SIZE,
options=_HIDDEN,
)
normal_mode: bpy.props.StringProperty(
default=NormalMode.NONE.value,
options=_HIDDEN,
)
refine_selection: bpy.props.BoolProperty(default=False, options=_HIDDEN)
def _needs_generation(self):
return (
self.refine_selection
or self.shape in DEPTH_CUTOUT_SHAPES
or NormalMode(self.normal_mode) != NormalMode.NONE
)
def execute(self, context):
try:
source_object = require_image_empty(self.source_object_name)
if video_total_frames(source_object.data) != 1:
raise ValueError("Cutout currently supports still images")
if self.shape not in CUTOUT_SHAPES:
raise ValueError(f"Unsupported Cutout shape: {self.shape}")
self.normal_mode = NormalMode(self.normal_mode).value
region_path = RegionPath.from_json(self.region_path_json)
source_rgba = image_rgba(source_object.data)
region_mask = rasterize_region_path(
tuple(source_object.data.size),
region_path,
source_alpha=source_rgba[:, :, 3],
alpha_threshold=CUTOUT_ALPHA_THRESHOLD,
antialias=True,
)
context.view_layer.objects.active = source_object
source_object.select_set(True)
self.max_input_size = configured_max_ai_input_size(context)
if not self._needs_generation():
region_to_shape(
context,
source_object,
self.shape,
self.mesh_detail,
region_mask,
source_rgba=source_rgba,
)
return {"FINISHED"}
setup_result = invoke_ai_setup_if_needed()
if setup_result is not None:
return setup_result
self.model, self.resolution_level = cutout_moge_settings(context)
self.region_bounds_json = json.dumps(
region_mask.bounds,
separators=(",", ":"),
)
self.input_path = str(
prepare_region_input(source_object.data, region_mask)
)
except (
ImageEditWarning,
OSError,
RuntimeError,
TypeError,
ValueError,
) as error:
report_image_edit_exception(self, error)
return {"CANCELLED"}
result = super().execute(context)
if "CANCELLED" in result:
cleanup_image_input(self.input_path, True)
return result
def request(self, context):
require_environment()
source_path = require_input_path(self.input_path)
output_depth = self.shape in DEPTH_CUTOUT_SHAPES
normal_mode = NormalMode(self.normal_mode)
if self.refine_selection:
require_model(BEN2_MODEL_KEY)
if output_depth or normal_mode != NormalMode.NONE:
require_model(self.model)
parameters = {
"input": str(source_path),
"device": production_device().lower(),
"refine_region": self.refine_selection,
"output_depth": output_depth,
"normal_mode": normal_mode.value,
"max_input_size": int(self.max_input_size),
}
if self.refine_selection:
parameters["refine_model"] = BEN2_MODEL_KEY
if output_depth or normal_mode != NormalMode.NONE:
start_frame = source_start_frame(self.source_object_name)
parameters.update(
model=self.model,
resolution_level=self.resolution_level,
)
parameters.update(
moge2_parameters(
context.scene.anyimage_settings,
source_path,
self.model,
self.resolution_level,
start_frame,
)
)
return parameters
def response(self, context, result):
source_object = require_image_empty(self.source_object_name)
source_bounds = tuple(json.loads(self.region_bounds_json))
region_mask = _generated_region_mask(
result,
self.input_path,
source_bounds,
)
value = result.value if isinstance(result.value, dict) else {}
normal_image = (
load_normal_result_image(result.file("normal"), source_object)
if "normal" in value
else None
)
field = (
load_image_geometry(result.file("image_geometry"))
if self.shape in DEPTH_CUTOUT_SHAPES
else None
)
region_to_shape(
context,
source_object,
self.shape,
self.mesh_detail,
region_mask,
field=field,
normal_image=normal_image,
normal_mode=NormalMode(self.normal_mode),
)
return f"{self.shape.replace('_', ' ').title()} is ready"
def cleanup(self):
cleanup_image_input(self.input_path, True)

136
src/anyimage/operators/cutout_tool/selection_tool.py → src/anyimage/operators/cutout_tool/select_region.py
View File

@@ -1,27 +1,42 @@
import json
import bpy
from ...properties import (
refine_selection_property,
)
from ..common.image import is_image_empty, video_total_frames
from ..common.image_edit import (
ImageEditWarning,
apply_image_mask_settings,
draw_image_mask_settings,
polygon_bounds,
from ..common.image_data import image_rgba, require_image_empty, video_total_frames
from ..common.image_interaction import (
apply_region_settings,
draw_region_settings,
report_image_edit_exception,
resolve_image_edit_click,
)
from ..common.image_interaction import hidden_string_property, serialize_matrix
from ..common.image_gesture import ImageMaskGesture, gesture_property
from ..common.image_mask import selected_image_rgba
from ..common.image_interaction import serialize_matrix
from ..common.image_interaction import ImageRegionGesture, gesture_property
from ..common.image_region import (
ImageEditWarning,
RegionPath,
rasterize_region_path,
)
CUTOUT_DETAIL_MIN = 4
CUTOUT_DETAIL_MAX = 1024
def _validate_cutout_region(
source_object,
region_path,
):
image = source_object.data
rgba = image_rgba(image)
rasterize_region_path(
tuple(image.size),
region_path,
source_alpha=rgba[:, :, 3],
alpha_threshold=0.1,
antialias=True,
)
def cutout_mesh_detail(image_size, point_spacing):
longest_edge = max(int(value) for value in image_size)
detail = round(longest_edge / max(int(point_spacing), 1))
@@ -34,23 +49,21 @@ def _pie_button(
label,
icon,
source_name,
selection_data,
region_path_json,
refine_selection,
normal_analysis,
invert,
normal_mode,
mesh_detail,
):
operator = pie.operator(
"anyimage.convert_cutout_shape",
"anyimage.cutout_region_to_shape",
text=label,
icon=icon,
)
operator.source_object_name = source_name
operator.selection_data = selection_data
operator.region_path_json = region_path_json
operator.shape = shape
operator.refine_selection = refine_selection
operator.normal_analysis = normal_analysis
operator.invert = invert
operator.normal_mode = normal_mode
operator.mesh_detail = mesh_detail
@@ -58,29 +71,27 @@ def _open_shape_pie(
context,
event,
source_object,
polygon,
region_path,
refine_selection=False,
invert=False,
):
source_name = source_object.name
selection_data = json.dumps(polygon, separators=(",", ":"))
region_path_json = region_path.to_json()
settings = context.scene.anyimage_settings
point_spacing = settings.cutout_point_spacing
normal_analysis = bool(getattr(settings, "cutout_normal_analysis", False))
normal_mode = str(getattr(settings, "cutout_normal_mode", "NONE"))
mesh_detail = cutout_mesh_detail(source_object.data.size, point_spacing)
def draw(menu, _context):
pie = menu.layout.menu_pie()
_pie_button(
pie,
"CUTOUT",
"SURFACE",
"Surface",
"MESH_GRID",
source_name,
selection_data,
region_path_json,
refine_selection,
normal_analysis,
invert,
normal_mode,
mesh_detail,
)
_pie_button(
@@ -89,10 +100,9 @@ def _open_shape_pie(
"Balloon",
"MESH_UVSPHERE",
source_name,
selection_data,
region_path_json,
refine_selection,
normal_analysis,
invert,
normal_mode,
mesh_detail,
)
_pie_button(
@@ -101,10 +111,9 @@ def _open_shape_pie(
"Depth Balloon",
"META_BALL",
source_name,
selection_data,
region_path_json,
refine_selection,
normal_analysis,
invert,
normal_mode,
mesh_detail,
)
_pie_button(
@@ -113,10 +122,9 @@ def _open_shape_pie(
"Depth Surface",
"SURFACE_NCURVE",
source_name,
selection_data,
region_path_json,
refine_selection,
normal_analysis,
invert,
normal_mode,
mesh_detail,
)
@@ -128,16 +136,16 @@ def _open_shape_pie(
)
class SelectCutoutRegion(ImageMaskGesture, bpy.types.Operator):
class SelectCutoutRegion(ImageRegionGesture, bpy.types.Operator):
bl_idname = "anyimage.select_cutout_region"
bl_label = "Select Cutout Region"
bl_description = "Lasso an Image Empty region, then choose its mesh shape"
bl_options = {"INTERNAL"}
active_tool_id = "anyimage.cutout_lasso"
source_object_name: hidden_string_property()
source_matrix_data: hidden_string_property()
selection_data: hidden_string_property()
source_object_name: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
source_matrix_data: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
region_path_json: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
refine_selection: refine_selection_property()
invert: bpy.props.BoolProperty(default=False, options={"SKIP_SAVE"})
gesture: gesture_property()
@@ -167,15 +175,15 @@ class SelectCutoutRegion(ImageMaskGesture, bpy.types.Operator):
if video_total_frames(source_object.data) > 1:
self.report({"WARNING"}, "Image tools only support a still image")
return {"CANCELLED"}
apply_image_mask_settings(self, context)
apply_region_settings(self, context)
self.invert = False
self.source_object_name = source_object.name
self.source_matrix_data = serialize_matrix(source_object.matrix_world)
width, height = (int(value) for value in source_object.data.size)
self.selection_data = json.dumps(
((0, 0), (width, 0), (width, height), (0, height)),
separators=(",", ":"),
)
self.region_path_json = RegionPath(
points=((0, 0), (width, 0), (width, height), (0, height)),
fill_rule="NONZERO",
).to_json()
return self.execute(context)
def modal(self, context, event):
@@ -183,47 +191,27 @@ class SelectCutoutRegion(ImageMaskGesture, bpy.types.Operator):
return super().modal(context, event)
def execute(self, context):
source_object = bpy.data.objects.get(self.source_object_name)
if not is_image_empty(source_object):
self.report({"ERROR"}, "The source Image Empty is no longer available")
try:
source_object = require_image_empty(self.source_object_name)
except RuntimeError as error:
self.report({"ERROR"}, str(error))
return {"CANCELLED"}
try:
polygon = json.loads(self.selection_data)
polygon_bounds(polygon)
selected_image_rgba(
source_object, polygon, self.invert, "NONZERO"
)
except (ImageEditWarning, json.JSONDecodeError, TypeError, ValueError) as error:
region_path = RegionPath.from_json(self.region_path_json)
_validate_cutout_region(source_object, region_path)
except (ImageEditWarning, TypeError, ValueError) as error:
report_image_edit_exception(self, error)
return {"CANCELLED"}
_open_shape_pie(
context,
self._pie_event,
source_object,
polygon,
region_path,
self.refine_selection,
self.invert,
)
return {"FINISHED"}
class ActivateCutoutTool(bpy.types.Operator):
bl_idname = "anyimage.activate_cutout_tool"
bl_label = "Cutout Tool"
bl_description = "Activate the Cutout Lasso tool"
bl_options = {"INTERNAL"}
@classmethod
def poll(cls, context):
return context.area is not None and context.area.type == "VIEW_3D"
def execute(self, _context):
return bpy.ops.wm.tool_set_by_id(
name=CutoutTool.bl_idname,
space_type="VIEW_3D",
)
class CutoutTool(bpy.types.WorkSpaceTool):
bl_space_type = "VIEW_3D"
bl_context_mode = "OBJECT"
@@ -248,5 +236,5 @@ class CutoutTool(bpy.types.WorkSpaceTool):
def draw_settings(context, layout, _tool):
settings = context.scene.anyimage_settings
layout.prop(settings, "cutout_point_spacing")
layout.prop(settings, "cutout_normal_analysis")
draw_image_mask_settings(context, layout, "cutout_refine_selection")
layout.prop(settings, "cutout_normal_mode")
draw_region_settings(context, layout, "cutout_refine_selection")

214
src/anyimage/operators/cutout_tool/shape_conversion.py
View File

@@ -1,214 +0,0 @@
import json
import bpy
from ...preferences import addon_preferences, configured_max_ai_input_size
from ...properties import BEN2_MODEL_KEY, DEFAULT_MOGE2_MODEL_KEY
from ...runtime import JobOperatorBase
from ...server.media.resolution import (
DEFAULT_MAX_AI_INPUT_SIZE,
MAX_MAX_AI_INPUT_SIZE,
MIN_MAX_AI_INPUT_SIZE,
)
from ..common.image import (
is_image_empty,
prepare_image_input,
remove_temporary_input,
video_total_frames,
)
from ..common.image_edit import report_image_edit_exception
from ..common.operator import (
HIDDEN,
cleanup_temporary_input,
input_path,
invoke_ai_setup_if_needed,
moge2_parameters,
production_device,
require_environment,
require_model,
source_start_frame,
)
from .build_shapes import build_cutout_response, build_local_cutout
from .shape_types import DEPTH_CUTOUT_SHAPES, LOCAL_CUTOUT_SHAPES
def cutout_moge_settings(context):
preferences = addon_preferences(context)
model = getattr(preferences, "mesh_depth_model", DEFAULT_MOGE2_MODEL_KEY)
if not model.startswith("MOGE2_"):
model = DEFAULT_MOGE2_MODEL_KEY
return (
model,
int(getattr(preferences, "mesh_resolution_level", 5)),
)
class GenerateCutoutShape(JobOperatorBase, bpy.types.Operator):
bl_idname = "anyimage.generate_cutout_shape"
bl_label = "Generate Cutout Shape"
job_type = "prepare-geometry-data"
starting_message = "Preparing Cutout shape"
input_path: bpy.props.StringProperty(options=HIDDEN)
source_object_name: bpy.props.StringProperty(options=HIDDEN)
shape: bpy.props.StringProperty(options=HIDDEN)
selection_data: bpy.props.StringProperty(options=HIDDEN)
mesh_detail: bpy.props.IntProperty(default=128, min=4, max=1024, options=HIDDEN)
model: bpy.props.StringProperty(default=DEFAULT_MOGE2_MODEL_KEY, options=HIDDEN)
resolution_level: bpy.props.IntProperty(default=5, min=0, max=9, options=HIDDEN)
max_input_size: bpy.props.IntProperty(
default=DEFAULT_MAX_AI_INPUT_SIZE,
min=MIN_MAX_AI_INPUT_SIZE,
max=MAX_MAX_AI_INPUT_SIZE,
options=HIDDEN,
)
normal_analysis: bpy.props.BoolProperty(default=False, options=HIDDEN)
delete_input: bpy.props.BoolProperty(default=False, options=HIDDEN)
refine_selection: bpy.props.BoolProperty(default=False, options=HIDDEN)
invert: bpy.props.BoolProperty(default=False, options=HIDDEN)
def request(self, context):
require_environment()
if self.shape in DEPTH_CUTOUT_SHAPES or self.normal_analysis:
require_model(self.model)
if self.refine_selection:
require_model(BEN2_MODEL_KEY)
polygon = json.loads(self.selection_data) if self.selection_data else None
if polygon is None or len(polygon) < 3:
raise ValueError("The Cutout selection is invalid")
parameters = {
"input": str(input_path(self.input_path)),
"analysis_input": str(input_path(self.input_path)),
"shape": self.shape,
"polygon": polygon,
"mesh_detail": self.mesh_detail,
"device": production_device().lower(),
"video_frames": 1,
"start_frame": source_start_frame(self.source_object_name),
"max_input_size": int(self.max_input_size),
"refine_selection": self.refine_selection,
"normal_analysis": self.normal_analysis,
"invert": self.invert,
}
if self.shape in DEPTH_CUTOUT_SHAPES or self.normal_analysis:
parameters.update(
model=self.model,
resolution_level=self.resolution_level,
)
if self.refine_selection:
parameters["refine_model"] = BEN2_MODEL_KEY
if self.shape not in DEPTH_CUTOUT_SHAPES:
return parameters
settings = context.scene.anyimage_settings
parameters.update(
moge2_parameters(
settings,
input_path(self.input_path),
self.model,
self.resolution_level,
source_start_frame(self.source_object_name),
)
)
parameters.update(shape=self.shape, polygon=polygon)
return parameters
def response(self, context, result):
build_cutout_response(context, result, self)
return f"{self.shape.replace('_', ' ').title()} is ready"
def cleanup(self):
cleanup_temporary_input(self.input_path, self.delete_input)
def start_cutout_job(
operator,
context,
shape,
selection_data,
refine_selection=False,
normal_analysis=False,
invert=False,
):
source_object = context.object
if video_total_frames(source_object.data) != 1:
operator.report({"ERROR"}, "Cutout currently supports still images")
return {"CANCELLED"}
setup_result = invoke_ai_setup_if_needed()
if setup_result is not None:
return setup_result
try:
source_path, delete_input = prepare_image_input(
source_object.data,
context.scene,
)
model, resolution_level = cutout_moge_settings(context)
except (RuntimeError, ValueError) as error:
operator.report({"ERROR"}, str(error))
return {"CANCELLED"}
result = bpy.ops.anyimage.generate_cutout_shape(
input_path=str(source_path),
source_object_name=source_object.name,
shape=shape,
selection_data=selection_data,
mesh_detail=operator.mesh_detail,
model=model,
resolution_level=resolution_level,
max_input_size=configured_max_ai_input_size(context),
normal_analysis=normal_analysis,
delete_input=delete_input,
refine_selection=refine_selection,
invert=invert,
)
if "CANCELLED" in result and delete_input:
remove_temporary_input(source_path)
return {"CANCELLED"} if "CANCELLED" in result else {"FINISHED"}
class ConvertCutoutShape(bpy.types.Operator):
bl_idname = "anyimage.convert_cutout_shape"
bl_label = "Convert Cutout Shape"
bl_description = "Convert the saved Cutout selection into the chosen shape"
bl_options = {"UNDO"}
source_object_name: bpy.props.StringProperty(options=HIDDEN)
selection_data: bpy.props.StringProperty(options=HIDDEN)
shape: bpy.props.StringProperty(options=HIDDEN)
mesh_detail: bpy.props.IntProperty(default=128, min=4, max=1024, options=HIDDEN)
refine_selection: bpy.props.BoolProperty(default=False, options=HIDDEN)
normal_analysis: bpy.props.BoolProperty(default=False, options=HIDDEN)
invert: bpy.props.BoolProperty(default=False, options=HIDDEN)
def execute(self, context):
source_object = bpy.data.objects.get(self.source_object_name)
if not is_image_empty(source_object):
self.report({"ERROR"}, "The source Image Empty is no longer available")
return {"CANCELLED"}
context.view_layer.objects.active = source_object
source_object.select_set(True)
if (
self.shape in LOCAL_CUTOUT_SHAPES
and not self.refine_selection
and not self.normal_analysis
):
try:
build_local_cutout(
self,
context,
source_object,
self.shape,
self.selection_data,
self.invert,
)
except Exception as error:
report_image_edit_exception(self, error)
return {"CANCELLED"}
return {"FINISHED"}
return start_cutout_job(
self,
context,
self.shape,
self.selection_data,
self.refine_selection,
self.normal_analysis,
self.invert,
)

23
src/anyimage/operators/cutout_tool/shape_types.py
View File

@@ -1,23 +0,0 @@
"""Cutout shape identifiers and Geometry Nodes asset names."""
CUTOUT_NODE_GROUP_NAME = "O Image Cutout"
CUTOUT_NODE_GROUP_NAMES = {
"CUTOUT": "O Image Surface",
"BALLOON": "O Image Balloon",
"DEPTH_BALLOON": "O Image Depth Balloon",
"DEPTH_SURFACE": "O Image Depth Surface",
}
CUTOUT_SHAPE_LABELS = {
"CUTOUT": "Surface",
"BALLOON": "Balloon",
"DEPTH_BALLOON": "Depth Balloon",
"DEPTH_SURFACE": "Depth Surface",
}
CUTOUT_SHAPE_INDICES = {
shape: index for index, shape in enumerate(CUTOUT_NODE_GROUP_NAMES)
}
CUTOUT_SHAPES = frozenset(CUTOUT_NODE_GROUP_NAMES)
LOCAL_CUTOUT_SHAPES = frozenset({"CUTOUT", "BALLOON"})
DEPTH_CUTOUT_SHAPES = frozenset({"DEPTH_BALLOON", "DEPTH_SURFACE"})
BALLOON_ATTRIBUTE_NAME = "o_balloon"