import math import bpy from ..common.image import ( MAX_PROJECTIVE_OUTPUT_PIXELS, PROJECTIVE_CHUNK_ROWS, create_image_edit_result, homography_from_points, image_empty_bounds, image_pixels, is_image_empty, premultiplied_rgba, replace_empty_image, sample_rgba, transform_homography, video_total_frames, ) from ..preferences import configured_max_frame_resolution from ..common.viewport import ( active_image_edit_source, active_view3d_tool_id, draw_dashed_line, draw_polygon_fill, drawing_in_region, image_edit_drag_keymap, image_edit_poll, rectangle_screen_path, report_image_edit_exception, ) from ..common.selection import ImageEditWarning, clip_polygon_halfplanes FRAME_TOOL_ID = "anyimage.frame" PROJECTIVE_RELATIVE_EPSILON = 1e-12 def _matrix_array(matrix): import numpy as np return np.asarray( tuple(tuple(float(value) for value in row) for row in matrix), dtype=np.float64, ) def frame_source_projection( projection_matrix, view_matrix, matrix_world, bounds, region_size, *, perspective, ): """Return normalized image-plane transforms for one frozen view.""" import numpy as np x_min, x_max, y_min, y_max = (float(value) for value in bounds) region_width, region_height = (float(value) for value in region_size) if min(x_max - x_min, y_max - y_min, region_width, region_height) <= 0.0: raise ValueError("The Image Empty projection is invalid") source_plane = np.asarray( ( (x_max - x_min, 0.0, x_min), (0.0, y_min - y_max, y_max), (0.0, 0.0, 0.0), (0.0, 0.0, 1.0), ), dtype=np.float64, ) world = _matrix_array(matrix_world) source_to_world = world @ source_plane source_to_view = _matrix_array(view_matrix) @ source_to_world source_to_clip = _matrix_array(projection_matrix) @ source_to_world source_to_screen = np.stack( ( 0.5 * (source_to_clip[0] + source_to_clip[3]), 0.5 * (source_to_clip[1] + source_to_clip[3]), source_to_clip[3], ) ) scale = float(np.max(np.abs(source_to_screen))) if not math.isfinite(scale) or scale <= 0.0: raise ValueError("The Image Empty projection is invalid") source_to_screen /= scale if not np.isfinite(source_to_screen).all(): raise ValueError("The Image Empty projection is invalid") condition = float(np.linalg.cond(source_to_screen)) if not math.isfinite(condition) or condition > 1.0 / np.finfo(np.float64).eps: raise ValueError("The Image Empty is edge-on to the current view") return { "bounds": (x_min, x_max, y_min, y_max), "origin_depth": -float((_matrix_array(view_matrix) @ world)[2, 3]), "perspective": bool(perspective), "region_size": (region_width, region_height), "screen_to_source": np.linalg.inv(source_to_screen), "source_to_screen": source_to_screen, "source_to_view": source_to_view, "source_to_world": source_to_world, } def _projective_points(points, matrix): """Map points and return finite results plus per-point validity.""" import numpy as np points = np.asarray(points, dtype=np.float64) flat = points.reshape((-1, 2)) homogeneous = np.column_stack((flat, np.ones(len(flat), dtype=np.float64))) mapped = homogeneous @ np.asarray(matrix, dtype=np.float64).T magnitude = np.max(np.abs(mapped), axis=1) valid = ( np.isfinite(mapped).all(axis=1) & (np.abs(mapped[:, 2]) > PROJECTIVE_RELATIVE_EPSILON * magnitude) ) result = np.full((len(flat), 2), np.nan, dtype=np.float64) result[valid] = mapped[valid, :2] / mapped[valid, 2, None] return result.reshape(points.shape), valid.reshape(points.shape[:-1]) def frame_source_coordinates(source, screen_points): """Return source pixels, view depth, validity, and normalized coordinates.""" import numpy as np screen_points = np.asarray(screen_points, dtype=np.float64) region_width, region_height = source["region_size"] normalized_screen = screen_points / (region_width, region_height) normalized_source, valid = _projective_points( normalized_screen, source["screen_to_source"], ) flat_source = normalized_source.reshape((-1, 2)) homogeneous = np.column_stack( (flat_source, np.ones(len(flat_source), dtype=np.float64)) ) depth = -(homogeneous @ source["source_to_view"][2]) depth = depth.reshape(valid.shape) valid &= np.isfinite(depth) if source["perspective"]: valid &= depth > 0.0 valid &= ( (normalized_source[..., 0] >= 0.0) & (normalized_source[..., 0] <= 1.0) & (normalized_source[..., 1] >= 0.0) & (normalized_source[..., 1] <= 1.0) ) image_width, image_height = source["image_size"] source_points = normalized_source * (image_width, image_height) source_points[~valid] = -1.0 return source_points, depth, valid, normalized_source def frame_source_overlap(source, frame_quad): """Clip the source plane before dividing its homogeneous projection.""" import numpy as np frame = np.asarray(frame_quad, dtype=np.float64) / source["region_size"] left, bottom = frame.min(axis=0) right, top = frame.max(axis=0) x, y, w = source["source_to_screen"] boundaries = [x - left * w, right * w - x, y - bottom * w, top * w - y] if source["perspective"]: boundaries.insert(0, -source["source_to_view"][2]) polygon = clip_polygon_halfplanes( ((0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0)), boundaries ) if len(polygon) < 3: return () screen, valid = _projective_points(polygon, source["source_to_screen"]) if not np.all(valid): return () screen *= source["region_size"] # Center the coordinates to avoid cancellation in a distant small polygon. centered = screen - screen[0] area = abs(np.sum(centered[:, 0] * np.roll(centered[:, 1], -1) - centered[:, 1] * np.roll(centered[:, 0], -1))) * 0.5 return tuple(map(tuple, screen)) if area > 1e-8 else () def polygon_centroid(polygon): """Return the area centroid of one non-degenerate screen polygon.""" cross_sum = 0.0 x_sum = 0.0 y_sum = 0.0 for first, second in zip(polygon, (*polygon[1:], polygon[0])): cross = first[0] * second[1] - second[0] * first[1] cross_sum += cross x_sum += (first[0] + second[0]) * cross y_sum += (first[1] + second[1]) * cross if abs(cross_sum) <= 1e-12: raise ValueError("Frame does not overlap the active Image Empty") return x_sum / (3.0 * cross_sum), y_sum / (3.0 * cross_sum) def _pixel_quad(image_size): width, height = (float(value) for value in image_size) if min(width, height) <= 0.0: raise ValueError("The source image has no readable dimensions") return ((0.0, 0.0), (width, 0.0), (width, height), (0.0, height)) def frame_output_size(frame_quad, region_size, max_resolution): """Choose output resolution from Frame pixels within the current Region.""" frame_width = abs(float(frame_quad[1][0] - frame_quad[0][0])) frame_height = abs(float(frame_quad[0][1] - frame_quad[3][1])) region_width, region_height = (float(value) for value in region_size) maximum = float(max_resolution) if min(frame_width, frame_height, region_width, region_height, maximum) <= 0.0: raise ValueError("The Frame output dimensions must be positive") scale = min( 1.0, region_width / frame_width, region_height / frame_height, maximum / max(frame_width, frame_height), math.sqrt( MAX_PROJECTIVE_OUTPUT_PIXELS / max(frame_width * frame_height, 1.0) ), ) return ( max(2, int(round(frame_width * scale))), max(2, int(round(frame_height * scale))), ) def _frame_source(source_object, region, view3d, view_matrix, *, active): import numpy as np image_size = tuple(source_object.data.size) pixels = image_pixels(source_object.data) width, height = image_size bounds = image_empty_bounds(source_object) source = frame_source_projection( view3d.perspective_matrix, view_matrix, source_object.matrix_world, bounds, (region.width, region.height), perspective=view3d.is_perspective, ) source.update( { "active": bool(active), "image_size": image_size, "name": source_object.name, "object": source_object, "pixels": premultiplied_rgba(pixels, image_size), "rgba": np.flipud(pixels.reshape((height, width, 4))), } ) return source def _composite_frame_samples(sources, screen_points): """Resolve independently sorted samples with an unweighted bottom RGB base.""" import numpy as np samples, depths, coordinates = [], [], [] for source in sources: points, depth, valid, _ = frame_source_coordinates(source, screen_points) sample = sample_rgba(source["pixels"], points) sample[~valid] = 0 depths.append(np.where(valid, depth, -math.inf)) samples.append(sample) coordinates.append(points) samples = np.stack(samples) depths = np.stack(depths) order = np.argsort(-depths, axis=0, kind="stable") bottom = order[0] covered = np.isfinite(np.take_along_axis(depths, bottom[None], axis=0)[0]) result = np.zeros_like(samples[0]) for index, source in enumerate(sources): selected = covered & (bottom == index) if np.any(selected): result[..., :3][selected] = sample_rgba( source["rgba"], coordinates[index][selected] )[..., :3] for index in range(len(sources)): layer = np.take_along_axis(samples, order[index][None, ..., None], axis=0)[0] alpha = layer[..., 3:4] result[..., 3:4] = alpha + result[..., 3:4] * (1 - alpha) if index: result[..., :3] = layer[..., :3] + result[..., :3] * (1 - alpha) return result, covered def _frame_boundary_pixels(sources, points, output_to_screen): """Conservatively detect projected contour and depth-order crossings.""" import numpy as np boundary = np.zeros(points.shape[:-1], dtype=bool) for source in sources: width, height = source["region_size"] mapping = source["screen_to_source"] @ np.diag((1 / width, 1 / height, 1)) @ output_to_screen u, v, w = mapping lines = [u, v, u - w, v - w, w] positive = np.ones_like(boundary) negative = np.ones_like(boundary) for line in (u, v, w - u, w - v, w): radius = 0.5 * (abs(line[0]) + abs(line[1])) value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2] positive &= value + radius >= 0 negative &= value - radius <= 0 source_boundary = np.zeros_like(boundary) if source["perspective"]: lines.append(source["source_to_view"][2] @ mapping) for line in lines: radius = 0.5 * (abs(line[0]) + abs(line[1])) value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2] source_boundary |= np.abs(value) < radius - 1e-10 * max(radius, 1e-20) boundary |= source_boundary & (positive | negative) if len(sources) > 1: first_order = None for offset in ((-0.499999, -0.499999), (0.499999, -0.499999), (-0.499999, 0.499999), (0.499999, 0.499999)): screen = transform_homography(points + offset, output_to_screen) depths = [] for source in sources: _, depth, valid, _ = frame_source_coordinates(source, screen) depths.append(np.where(valid, depth, -math.inf)) order = np.argsort(-np.stack(depths), axis=0, kind="stable") if first_order is None: first_order = order else: boundary |= np.any(order != first_order, axis=0) return boundary def composite_frame_pixels(sources, frame_quad, output_size): """Composite projected Image Empty sources into one Blender-order RGBA image.""" import numpy as np output_width, output_height = (int(value) for value in output_size) if min(output_width, output_height) <= 0: raise ValueError("The Frame output dimensions must be positive") sources = tuple( sorted(sources, key=lambda source: (source["active"], source["name"])) ) if not sources or not sources[-1]["active"]: raise ValueError("Frame requires one active Image Empty") output_quad = _pixel_quad(output_size) output_to_screen = homography_from_points(output_quad, frame_quad) output = np.zeros((output_height, output_width, 4), dtype=np.float32) covered = np.zeros((output_height, output_width), dtype=bool) target_x = np.arange(output_width, dtype=np.float64) + 0.5 for row_start in range(0, output_height, PROJECTIVE_CHUNK_ROWS): row_end = min(row_start + PROJECTIVE_CHUNK_ROWS, output_height) target_y = np.arange(row_start, row_end, dtype=np.float64) + 0.5 x, y = np.meshgrid(target_x, target_y) points = np.stack((x, y), axis=-1) composite, coverage = _composite_frame_samples( sources, transform_homography(points, output_to_screen) ) boundary = _frame_boundary_pixels(sources, points, output_to_screen) if np.any(boundary): edge_points = points[boundary] accumulated = np.zeros((len(edge_points), 4), dtype=np.float32) counts = np.zeros(len(edge_points), dtype=np.int32) for dy in (-0.375, -0.125, 0.125, 0.375): for dx in (-0.375, -0.125, 0.125, 0.375): sample, hit = _composite_frame_samples( sources, transform_homography(edge_points + (dx, dy), output_to_screen), ) accumulated += sample counts += hit accumulated[..., 3] /= 16 accumulated[..., :3] /= np.maximum(counts, 1)[..., None] composite[boundary] = accumulated coverage[boundary] = counts > 0 covered[row_start:row_end] = coverage output[row_start:row_end] = composite output[..., 3][output[..., 3] <= 1e-8] = 0.0 output[~covered] = 0.0 return np.flipud(output).ravel() def view_facing_frame_geometry(corners, output_size): """Return a world matrix array and display size for frame-plane corners.""" import numpy as np corners = np.asarray(corners, dtype=np.float64) if corners.shape != (4, 3) or not np.isfinite(corners).all(): raise ValueError("The Frame view plane is invalid") top_left, top_right, _bottom_right, bottom_left = corners right = top_right - top_left up = top_left - bottom_left width = float(np.linalg.norm(right)) height = float(np.linalg.norm(up)) if min(width, height) <= 1e-10: raise ValueError("Draw a larger Frame") right /= width up /= height forward = np.cross(right, up) forward_length = float(np.linalg.norm(forward)) if forward_length <= 1e-10: raise ValueError("The Frame view plane is invalid") forward /= forward_length matrix = np.identity(4, dtype=np.float64) matrix[:3, 0] = right matrix[:3, 1] = up matrix[:3, 2] = forward matrix[:3, 3] = corners.mean(axis=0) display_size = width if output_size[0] >= output_size[1] else height return matrix, display_size def frame_result_transform(frame_quad, source_origin, region, view3d, output_size): """Return the view-facing object matrix and Image Empty display size.""" from mathutils import Matrix from bpy_extras import view3d_utils corners = tuple( tuple( float(value) for value in view3d_utils.region_2d_to_location_3d( region, view3d, point, source_origin, ) ) for point in frame_quad ) matrix, display_size = view_facing_frame_geometry(corners, output_size) return Matrix(tuple(tuple(row) for row in matrix)), display_size def frame_depth_location(active_source, overlap_polygon): """Return the world point whose view depth places the Frame result.""" import numpy as np source_object = active_source["object"] if not active_source["perspective"] or active_source["origin_depth"] > 0.0: return source_object.matrix_world.translation.copy() centroid = np.asarray((polygon_centroid(overlap_polygon),), dtype=np.float64) _points, depth, valid, normalized_source = frame_source_coordinates( active_source, centroid, ) if not bool(valid[0]) or depth[0] <= 0.0: raise ValueError("The Frame result depth is invalid") source_point = np.asarray( (normalized_source[0, 0], normalized_source[0, 1], 1.0), dtype=np.float64, ) world = active_source["source_to_world"] @ source_point if not np.isfinite(world).all() or abs(world[3]) <= 1e-12: raise ValueError("The Frame result depth is invalid") return tuple(float(value) for value in world[:3] / world[3]) def selected_frame_sources(context): """Return selected still Image Empties with the active object last.""" active = active_image_edit_source(context) sources = [ source for source in context.selected_objects if is_image_empty(source) ] for source in sources: if video_total_frames(source.data) > 1: raise ValueError("Frame only supports still images") return tuple( sorted(sources, key=lambda source: (source is active, source.name)) ) def apply_frame(context, source_objects, frame_quad): """Bake selected projections into the active Image Empty.""" active = context.active_object if active not in source_objects: raise ValueError("The active Image Empty is not a Frame source") view3d = context.region_data sources = tuple( _frame_source( source, context.region, view3d, view3d.view_matrix, active=source is active, ) for source in source_objects ) active_source = next(source for source in sources if source["active"]) active_overlap = frame_source_overlap(active_source, frame_quad) if not active_overlap: raise ImageEditWarning("Frame does not overlap the active Image Empty") output_size = frame_output_size( frame_quad, (context.region.width, context.region.height), configured_max_frame_resolution(context), ) pixels = composite_frame_pixels(sources, frame_quad, output_size) depth_location = frame_depth_location(active_source, active_overlap) matrix, display_size = frame_result_transform( frame_quad, depth_location, context.region, view3d, output_size, ) result_image = create_image_edit_result(active.data, pixels, output_size) old_matrix = active.matrix_world.copy() old_display_size = active.empty_display_size old_offset = tuple(active.empty_image_offset) source_images = [] for source in source_objects: if all(image is not source.data for image in source_images): source_images.append(source.data) try: active.matrix_world = matrix active.empty_display_size = display_size active.empty_image_offset = (-0.5, -0.5) replace_empty_image( active, result_image, removed_objects=tuple(source for source in source_objects if source != active), ) except Exception: active.matrix_world = old_matrix active.empty_display_size = old_display_size active.empty_image_offset = old_offset raise for source in source_objects: if source is not active: bpy.data.objects.remove(source, do_unlink=True) for image in source_images: try: if image.users == 0 and not image.use_fake_user: bpy.data.images.remove(image, do_unlink=True) except (ReferenceError, RuntimeError): pass return active class FrameImages(bpy.types.Operator): bl_idname = "anyimage.frame_images" bl_label = "Frame Images" bl_description = "Bake selected Image Empties into a view-facing frame" bl_options = {"UNDO"} _active_name = "" _source_names = () _start = None _cursor = None _handle = None _area_pointer = 0 _region_pointer = 0 @classmethod def poll(cls, context): return image_edit_poll(context) def _draw_overlay(self): if not drawing_in_region(self._area_pointer, self._region_pointer): return frame = rectangle_screen_path(self._start, self._cursor) if not frame: return draw_polygon_fill(frame) draw_dashed_line((*frame, frame[0])) def _finish(self, context): if self._handle is not None: bpy.types.SpaceView3D.draw_handler_remove(self._handle, "WINDOW") self._handle = None context.workspace.status_text_set(None) if context.area is not None: context.area.tag_redraw() def cancel(self, context): self._finish(context) def invoke(self, context, event): try: sources = selected_frame_sources(context) except (TypeError, ValueError) as error: report_image_edit_exception(self, error) return {"CANCELLED"} self._active_name = context.active_object.name self._source_names = tuple(source.name for source in sources) point = (float(event.mouse_region_x), float(event.mouse_region_y)) self._start = point self._cursor = point self._area_pointer = context.area.as_pointer() self._region_pointer = context.region.as_pointer() self._handle = bpy.types.SpaceView3D.draw_handler_add( self._draw_overlay, (), "WINDOW", "POST_PIXEL" ) context.workspace.status_text_set( "Frame: drag a rectangle to merge selected Image Empties; " "RMB or Esc cancels" ) context.window_manager.modal_handler_add(self) context.area.tag_redraw() return {"RUNNING_MODAL"} def modal(self, context, event): active_tool = active_view3d_tool_id(context) if active_tool is not None and active_tool != FRAME_TOOL_ID: self._finish(context) return {"CANCELLED"} if event.type in {"ESC", "RIGHTMOUSE"} and event.value == "PRESS": self._finish(context) return {"CANCELLED"} if event.type == "MOUSEMOVE": self._cursor = ( float(event.mouse_region_x), float(event.mouse_region_y), ) context.area.tag_redraw() return {"RUNNING_MODAL"} if event.type == "LEFTMOUSE" and event.value == "RELEASE": self._cursor = ( float(event.mouse_region_x), float(event.mouse_region_y), ) frame = rectangle_screen_path(self._start, self._cursor) if not frame: self._finish(context) return {"CANCELLED"} try: active = bpy.data.objects.get(self._active_name) sources = tuple( bpy.data.objects.get(name) for name in self._source_names ) if ( not is_image_empty(active) or any(not is_image_empty(source) for source in sources) or context.active_object is not active ): raise ValueError("The selected Image Empties changed during Frame") apply_frame(context, sources, frame) except (RuntimeError, TypeError, ValueError) as error: report_image_edit_exception(self, error) self._finish(context) return {"CANCELLED"} self._finish(context) return {"FINISHED"} return {"PASS_THROUGH"} class FrameTool(bpy.types.WorkSpaceTool): bl_space_type = "VIEW_3D" bl_context_mode = "OBJECT" bl_idname = FRAME_TOOL_ID bl_label = "Frame" bl_description = "Bake selected Image Empties into a view-facing frame" bl_icon = "ops.sculpt.border_mask" bl_operator = FrameImages.bl_idname bl_keymap = image_edit_drag_keymap(FrameImages.bl_idname)