Files
AnyImage/src/anyimage/operators/frame_tool.py
nan f5aff50b13 Refactor and clean up codebase by removing unused functions and reorganizing imports
- Removed unused model keys and functions from properties.py.
- Deleted projection.py as its functions were no longer needed.
- Cleaned up model_catalog.py by removing unused model directory function.
- Updated imports in various test files to reflect new module structure.
- Removed deprecated test cases related to image editing tools.
- Added new tests for ONNX runtime provider selection logic.
2026-09-09 10:31:56 +08:00

649 lines
24 KiB
Python

import math
import bpy
from ..common.image import (
MAX_PROJECTIVE_OUTPUT_PIXELS,
PROJECTIVE_CHUNK_ROWS,
create_image_edit_result,
homography_from_points,
image_empty_bounds,
image_pixels,
is_image_empty,
premultiplied_rgba,
replace_empty_image,
sample_rgba,
transform_homography,
video_total_frames,
)
from ..preferences import configured_max_frame_resolution
from ..common.viewport import (
active_image_edit_source,
active_view3d_tool_id,
draw_dashed_line,
draw_polygon_fill,
drawing_in_region,
image_edit_drag_keymap,
image_edit_poll,
rectangle_screen_path,
report_image_edit_exception,
)
from ..common.selection import ImageEditWarning, clip_polygon_halfplanes
FRAME_TOOL_ID = "anyimage.frame"
PROJECTIVE_RELATIVE_EPSILON = 1e-12
def _matrix_array(matrix):
import numpy as np
return np.asarray(
tuple(tuple(float(value) for value in row) for row in matrix),
dtype=np.float64,
)
def frame_source_projection(
projection_matrix,
view_matrix,
matrix_world,
bounds,
region_size,
*,
perspective,
):
"""Return normalized image-plane transforms for one frozen view."""
import numpy as np
x_min, x_max, y_min, y_max = (float(value) for value in bounds)
region_width, region_height = (float(value) for value in region_size)
if min(x_max - x_min, y_max - y_min, region_width, region_height) <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_plane = np.asarray(
(
(x_max - x_min, 0.0, x_min),
(0.0, y_min - y_max, y_max),
(0.0, 0.0, 0.0),
(0.0, 0.0, 1.0),
),
dtype=np.float64,
)
world = _matrix_array(matrix_world)
source_to_world = world @ source_plane
source_to_view = _matrix_array(view_matrix) @ source_to_world
source_to_clip = _matrix_array(projection_matrix) @ source_to_world
source_to_screen = np.stack(
(
0.5 * (source_to_clip[0] + source_to_clip[3]),
0.5 * (source_to_clip[1] + source_to_clip[3]),
source_to_clip[3],
)
)
scale = float(np.max(np.abs(source_to_screen)))
if not math.isfinite(scale) or scale <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_to_screen /= scale
if not np.isfinite(source_to_screen).all():
raise ValueError("The Image Empty projection is invalid")
condition = float(np.linalg.cond(source_to_screen))
if not math.isfinite(condition) or condition > 1.0 / np.finfo(np.float64).eps:
raise ValueError("The Image Empty is edge-on to the current view")
return {
"bounds": (x_min, x_max, y_min, y_max),
"origin_depth": -float((_matrix_array(view_matrix) @ world)[2, 3]),
"perspective": bool(perspective),
"region_size": (region_width, region_height),
"screen_to_source": np.linalg.inv(source_to_screen),
"source_to_screen": source_to_screen,
"source_to_view": source_to_view,
"source_to_world": source_to_world,
}
def _projective_points(points, matrix):
"""Map points and return finite results plus per-point validity."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
flat = points.reshape((-1, 2))
homogeneous = np.column_stack((flat, np.ones(len(flat), dtype=np.float64)))
mapped = homogeneous @ np.asarray(matrix, dtype=np.float64).T
magnitude = np.max(np.abs(mapped), axis=1)
valid = (
np.isfinite(mapped).all(axis=1)
& (np.abs(mapped[:, 2]) > PROJECTIVE_RELATIVE_EPSILON * magnitude)
)
result = np.full((len(flat), 2), np.nan, dtype=np.float64)
result[valid] = mapped[valid, :2] / mapped[valid, 2, None]
return result.reshape(points.shape), valid.reshape(points.shape[:-1])
def frame_source_coordinates(source, screen_points):
"""Return source pixels, view depth, validity, and normalized coordinates."""
import numpy as np
screen_points = np.asarray(screen_points, dtype=np.float64)
region_width, region_height = source["region_size"]
normalized_screen = screen_points / (region_width, region_height)
normalized_source, valid = _projective_points(
normalized_screen,
source["screen_to_source"],
)
flat_source = normalized_source.reshape((-1, 2))
homogeneous = np.column_stack(
(flat_source, np.ones(len(flat_source), dtype=np.float64))
)
depth = -(homogeneous @ source["source_to_view"][2])
depth = depth.reshape(valid.shape)
valid &= np.isfinite(depth)
if source["perspective"]:
valid &= depth > 0.0
valid &= (
(normalized_source[..., 0] >= 0.0)
& (normalized_source[..., 0] <= 1.0)
& (normalized_source[..., 1] >= 0.0)
& (normalized_source[..., 1] <= 1.0)
)
image_width, image_height = source["image_size"]
source_points = normalized_source * (image_width, image_height)
source_points[~valid] = -1.0
return source_points, depth, valid, normalized_source
def frame_source_overlap(source, frame_quad):
"""Clip the source plane before dividing its homogeneous projection."""
import numpy as np
frame = np.asarray(frame_quad, dtype=np.float64) / source["region_size"]
left, bottom = frame.min(axis=0)
right, top = frame.max(axis=0)
x, y, w = source["source_to_screen"]
boundaries = [x - left * w, right * w - x, y - bottom * w, top * w - y]
if source["perspective"]:
boundaries.insert(0, -source["source_to_view"][2])
polygon = clip_polygon_halfplanes(
((0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0)), boundaries
)
if len(polygon) < 3:
return ()
screen, valid = _projective_points(polygon, source["source_to_screen"])
if not np.all(valid):
return ()
screen *= source["region_size"]
# Center the coordinates to avoid cancellation in a distant small polygon.
centered = screen - screen[0]
area = abs(np.sum(centered[:, 0] * np.roll(centered[:, 1], -1)
- centered[:, 1] * np.roll(centered[:, 0], -1))) * 0.5
return tuple(map(tuple, screen)) if area > 1e-8 else ()
def polygon_centroid(polygon):
"""Return the area centroid of one non-degenerate screen polygon."""
cross_sum = 0.0
x_sum = 0.0
y_sum = 0.0
for first, second in zip(polygon, (*polygon[1:], polygon[0])):
cross = first[0] * second[1] - second[0] * first[1]
cross_sum += cross
x_sum += (first[0] + second[0]) * cross
y_sum += (first[1] + second[1]) * cross
if abs(cross_sum) <= 1e-12:
raise ValueError("Frame does not overlap the active Image Empty")
return x_sum / (3.0 * cross_sum), y_sum / (3.0 * cross_sum)
def _pixel_quad(image_size):
width, height = (float(value) for value in image_size)
if min(width, height) <= 0.0:
raise ValueError("The source image has no readable dimensions")
return ((0.0, 0.0), (width, 0.0), (width, height), (0.0, height))
def frame_output_size(frame_quad, region_size, max_resolution):
"""Choose output resolution from Frame pixels within the current Region."""
frame_width = abs(float(frame_quad[1][0] - frame_quad[0][0]))
frame_height = abs(float(frame_quad[0][1] - frame_quad[3][1]))
region_width, region_height = (float(value) for value in region_size)
maximum = float(max_resolution)
if min(frame_width, frame_height, region_width, region_height, maximum) <= 0.0:
raise ValueError("The Frame output dimensions must be positive")
scale = min(
1.0,
region_width / frame_width,
region_height / frame_height,
maximum / max(frame_width, frame_height),
math.sqrt(
MAX_PROJECTIVE_OUTPUT_PIXELS / max(frame_width * frame_height, 1.0)
),
)
return (
max(2, int(round(frame_width * scale))),
max(2, int(round(frame_height * scale))),
)
def _frame_source(source_object, region, view3d, view_matrix, *, active):
import numpy as np
image_size = tuple(source_object.data.size)
pixels = image_pixels(source_object.data)
width, height = image_size
bounds = image_empty_bounds(source_object)
source = frame_source_projection(
view3d.perspective_matrix,
view_matrix,
source_object.matrix_world,
bounds,
(region.width, region.height),
perspective=view3d.is_perspective,
)
source.update(
{
"active": bool(active),
"image_size": image_size,
"name": source_object.name,
"object": source_object,
"pixels": premultiplied_rgba(pixels, image_size),
"rgba": np.flipud(pixels.reshape((height, width, 4))),
}
)
return source
def _composite_frame_samples(sources, screen_points):
"""Resolve independently sorted samples with an unweighted bottom RGB base."""
import numpy as np
samples, depths, coordinates = [], [], []
for source in sources:
points, depth, valid, _ = frame_source_coordinates(source, screen_points)
sample = sample_rgba(source["pixels"], points)
sample[~valid] = 0
depths.append(np.where(valid, depth, -math.inf))
samples.append(sample)
coordinates.append(points)
samples = np.stack(samples)
depths = np.stack(depths)
order = np.argsort(-depths, axis=0, kind="stable")
bottom = order[0]
covered = np.isfinite(np.take_along_axis(depths, bottom[None], axis=0)[0])
result = np.zeros_like(samples[0])
for index, source in enumerate(sources):
selected = covered & (bottom == index)
if np.any(selected):
result[..., :3][selected] = sample_rgba(
source["rgba"], coordinates[index][selected]
)[..., :3]
for index in range(len(sources)):
layer = np.take_along_axis(samples, order[index][None, ..., None], axis=0)[0]
alpha = layer[..., 3:4]
result[..., 3:4] = alpha + result[..., 3:4] * (1 - alpha)
if index:
result[..., :3] = layer[..., :3] + result[..., :3] * (1 - alpha)
return result, covered
def _frame_boundary_pixels(sources, points, output_to_screen):
"""Conservatively detect projected contour and depth-order crossings."""
import numpy as np
boundary = np.zeros(points.shape[:-1], dtype=bool)
for source in sources:
width, height = source["region_size"]
mapping = source["screen_to_source"] @ np.diag((1 / width, 1 / height, 1)) @ output_to_screen
u, v, w = mapping
lines = [u, v, u - w, v - w, w]
positive = np.ones_like(boundary)
negative = np.ones_like(boundary)
for line in (u, v, w - u, w - v, w):
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
positive &= value + radius >= 0
negative &= value - radius <= 0
source_boundary = np.zeros_like(boundary)
if source["perspective"]:
lines.append(source["source_to_view"][2] @ mapping)
for line in lines:
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
source_boundary |= np.abs(value) < radius - 1e-10 * max(radius, 1e-20)
boundary |= source_boundary & (positive | negative)
if len(sources) > 1:
first_order = None
for offset in ((-0.499999, -0.499999), (0.499999, -0.499999),
(-0.499999, 0.499999), (0.499999, 0.499999)):
screen = transform_homography(points + offset, output_to_screen)
depths = []
for source in sources:
_, depth, valid, _ = frame_source_coordinates(source, screen)
depths.append(np.where(valid, depth, -math.inf))
order = np.argsort(-np.stack(depths), axis=0, kind="stable")
if first_order is None:
first_order = order
else:
boundary |= np.any(order != first_order, axis=0)
return boundary
def composite_frame_pixels(sources, frame_quad, output_size):
"""Composite projected Image Empty sources into one Blender-order RGBA image."""
import numpy as np
output_width, output_height = (int(value) for value in output_size)
if min(output_width, output_height) <= 0:
raise ValueError("The Frame output dimensions must be positive")
sources = tuple(
sorted(sources, key=lambda source: (source["active"], source["name"]))
)
if not sources or not sources[-1]["active"]:
raise ValueError("Frame requires one active Image Empty")
output_quad = _pixel_quad(output_size)
output_to_screen = homography_from_points(output_quad, frame_quad)
output = np.zeros((output_height, output_width, 4), dtype=np.float32)
covered = np.zeros((output_height, output_width), dtype=bool)
target_x = np.arange(output_width, dtype=np.float64) + 0.5
for row_start in range(0, output_height, PROJECTIVE_CHUNK_ROWS):
row_end = min(row_start + PROJECTIVE_CHUNK_ROWS, output_height)
target_y = np.arange(row_start, row_end, dtype=np.float64) + 0.5
x, y = np.meshgrid(target_x, target_y)
points = np.stack((x, y), axis=-1)
composite, coverage = _composite_frame_samples(
sources, transform_homography(points, output_to_screen)
)
boundary = _frame_boundary_pixels(sources, points, output_to_screen)
if np.any(boundary):
edge_points = points[boundary]
accumulated = np.zeros((len(edge_points), 4), dtype=np.float32)
counts = np.zeros(len(edge_points), dtype=np.int32)
for dy in (-0.375, -0.125, 0.125, 0.375):
for dx in (-0.375, -0.125, 0.125, 0.375):
sample, hit = _composite_frame_samples(
sources,
transform_homography(edge_points + (dx, dy), output_to_screen),
)
accumulated += sample
counts += hit
accumulated[..., 3] /= 16
accumulated[..., :3] /= np.maximum(counts, 1)[..., None]
composite[boundary] = accumulated
coverage[boundary] = counts > 0
covered[row_start:row_end] = coverage
output[row_start:row_end] = composite
output[..., 3][output[..., 3] <= 1e-8] = 0.0
output[~covered] = 0.0
return np.flipud(output).ravel()
def view_facing_frame_geometry(corners, output_size):
"""Return a world matrix array and display size for frame-plane corners."""
import numpy as np
corners = np.asarray(corners, dtype=np.float64)
if corners.shape != (4, 3) or not np.isfinite(corners).all():
raise ValueError("The Frame view plane is invalid")
top_left, top_right, _bottom_right, bottom_left = corners
right = top_right - top_left
up = top_left - bottom_left
width = float(np.linalg.norm(right))
height = float(np.linalg.norm(up))
if min(width, height) <= 1e-10:
raise ValueError("Draw a larger Frame")
right /= width
up /= height
forward = np.cross(right, up)
forward_length = float(np.linalg.norm(forward))
if forward_length <= 1e-10:
raise ValueError("The Frame view plane is invalid")
forward /= forward_length
matrix = np.identity(4, dtype=np.float64)
matrix[:3, 0] = right
matrix[:3, 1] = up
matrix[:3, 2] = forward
matrix[:3, 3] = corners.mean(axis=0)
display_size = width if output_size[0] >= output_size[1] else height
return matrix, display_size
def frame_result_transform(frame_quad, source_origin, region, view3d, output_size):
"""Return the view-facing object matrix and Image Empty display size."""
from mathutils import Matrix
from bpy_extras import view3d_utils
corners = tuple(
tuple(
float(value)
for value in view3d_utils.region_2d_to_location_3d(
region,
view3d,
point,
source_origin,
)
)
for point in frame_quad
)
matrix, display_size = view_facing_frame_geometry(corners, output_size)
return Matrix(tuple(tuple(row) for row in matrix)), display_size
def frame_depth_location(active_source, overlap_polygon):
"""Return the world point whose view depth places the Frame result."""
import numpy as np
source_object = active_source["object"]
if not active_source["perspective"] or active_source["origin_depth"] > 0.0:
return source_object.matrix_world.translation.copy()
centroid = np.asarray((polygon_centroid(overlap_polygon),), dtype=np.float64)
_points, depth, valid, normalized_source = frame_source_coordinates(
active_source,
centroid,
)
if not bool(valid[0]) or depth[0] <= 0.0:
raise ValueError("The Frame result depth is invalid")
source_point = np.asarray(
(normalized_source[0, 0], normalized_source[0, 1], 1.0),
dtype=np.float64,
)
world = active_source["source_to_world"] @ source_point
if not np.isfinite(world).all() or abs(world[3]) <= 1e-12:
raise ValueError("The Frame result depth is invalid")
return tuple(float(value) for value in world[:3] / world[3])
def selected_frame_sources(context):
"""Return selected still Image Empties with the active object last."""
active = active_image_edit_source(context)
sources = [
source for source in context.selected_objects if is_image_empty(source)
]
for source in sources:
if video_total_frames(source.data) > 1:
raise ValueError("Frame only supports still images")
return tuple(
sorted(sources, key=lambda source: (source is active, source.name))
)
def apply_frame(context, source_objects, frame_quad):
"""Bake selected projections into the active Image Empty."""
active = context.active_object
if active not in source_objects:
raise ValueError("The active Image Empty is not a Frame source")
view3d = context.region_data
sources = tuple(
_frame_source(
source,
context.region,
view3d,
view3d.view_matrix,
active=source is active,
)
for source in source_objects
)
active_source = next(source for source in sources if source["active"])
active_overlap = frame_source_overlap(active_source, frame_quad)
if not active_overlap:
raise ImageEditWarning("Frame does not overlap the active Image Empty")
output_size = frame_output_size(
frame_quad,
(context.region.width, context.region.height),
configured_max_frame_resolution(context),
)
pixels = composite_frame_pixels(sources, frame_quad, output_size)
depth_location = frame_depth_location(active_source, active_overlap)
matrix, display_size = frame_result_transform(
frame_quad,
depth_location,
context.region,
view3d,
output_size,
)
result_image = create_image_edit_result(active.data, pixels, output_size)
old_matrix = active.matrix_world.copy()
old_display_size = active.empty_display_size
old_offset = tuple(active.empty_image_offset)
source_images = []
for source in source_objects:
if all(image is not source.data for image in source_images):
source_images.append(source.data)
try:
active.matrix_world = matrix
active.empty_display_size = display_size
active.empty_image_offset = (-0.5, -0.5)
replace_empty_image(
active, result_image,
removed_objects=tuple(source for source in source_objects if source != active),
)
except Exception:
active.matrix_world = old_matrix
active.empty_display_size = old_display_size
active.empty_image_offset = old_offset
raise
for source in source_objects:
if source is not active:
bpy.data.objects.remove(source, do_unlink=True)
for image in source_images:
try:
if image.users == 0 and not image.use_fake_user:
bpy.data.images.remove(image, do_unlink=True)
except (ReferenceError, RuntimeError):
pass
return active
class FrameImages(bpy.types.Operator):
bl_idname = "anyimage.frame_images"
bl_label = "Frame Images"
bl_description = "Bake selected Image Empties into a view-facing frame"
bl_options = {"UNDO"}
_active_name = ""
_source_names = ()
_start = None
_cursor = None
_handle = None
_area_pointer = 0
_region_pointer = 0
@classmethod
def poll(cls, context):
return image_edit_poll(context)
def _draw_overlay(self):
if not drawing_in_region(self._area_pointer, self._region_pointer):
return
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
return
draw_polygon_fill(frame)
draw_dashed_line((*frame, frame[0]))
def _finish(self, context):
if self._handle is not None:
bpy.types.SpaceView3D.draw_handler_remove(self._handle, "WINDOW")
self._handle = None
context.workspace.status_text_set(None)
if context.area is not None:
context.area.tag_redraw()
def cancel(self, context):
self._finish(context)
def invoke(self, context, event):
try:
sources = selected_frame_sources(context)
except (TypeError, ValueError) as error:
report_image_edit_exception(self, error)
return {"CANCELLED"}
self._active_name = context.active_object.name
self._source_names = tuple(source.name for source in sources)
point = (float(event.mouse_region_x), float(event.mouse_region_y))
self._start = point
self._cursor = point
self._area_pointer = context.area.as_pointer()
self._region_pointer = context.region.as_pointer()
self._handle = bpy.types.SpaceView3D.draw_handler_add(
self._draw_overlay, (), "WINDOW", "POST_PIXEL"
)
context.workspace.status_text_set(
"Frame: drag a rectangle to merge selected Image Empties; "
"RMB or Esc cancels"
)
context.window_manager.modal_handler_add(self)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
def modal(self, context, event):
active_tool = active_view3d_tool_id(context)
if active_tool is not None and active_tool != FRAME_TOOL_ID:
self._finish(context)
return {"CANCELLED"}
if event.type in {"ESC", "RIGHTMOUSE"} and event.value == "PRESS":
self._finish(context)
return {"CANCELLED"}
if event.type == "MOUSEMOVE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
if event.type == "LEFTMOUSE" and event.value == "RELEASE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
self._finish(context)
return {"CANCELLED"}
try:
active = bpy.data.objects.get(self._active_name)
sources = tuple(
bpy.data.objects.get(name) for name in self._source_names
)
if (
not is_image_empty(active)
or any(not is_image_empty(source) for source in sources)
or context.active_object is not active
):
raise ValueError("The selected Image Empties changed during Frame")
apply_frame(context, sources, frame)
except (RuntimeError, TypeError, ValueError) as error:
report_image_edit_exception(self, error)
self._finish(context)
return {"CANCELLED"}
self._finish(context)
return {"FINISHED"}
return {"PASS_THROUGH"}
class FrameTool(bpy.types.WorkSpaceTool):
bl_space_type = "VIEW_3D"
bl_context_mode = "OBJECT"
bl_idname = FRAME_TOOL_ID
bl_label = "Frame"
bl_description = "Bake selected Image Empties into a view-facing frame"
bl_icon = "ops.sculpt.border_mask"
bl_operator = FrameImages.bl_idname
bl_keymap = image_edit_drag_keymap(FrameImages.bl_idname)