icrdr/AnyImage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add new functionality for image rectification and depth metadata handling

Browse Source 
- Implemented a preview tool for rectifying images with interactive aspect controls in `preview.py`.
- Added tests for depth metadata loading to ensure consistent schema usage in `test_depth_metadata.py`.
- Developed comprehensive tests for cutout calibration processes in `test_cutout_calibration.py`, including stability checks and depth calibration.
- Created tests for frame compositing operations in `test_frame_compositing.py`, ensuring correct handling of depth and transparency.
- Introduced material slot inheritance functionality in `material_slots.py` to manage geometry material assignments.
- Built shared shader nodes for material processing in `shader.py`, including roughness and bump mapping setups.
This commit is contained in:
马南
2026-09-12 19:20:06 +08:00
parent e8268504c2
commit 938d95db0f
175 changed files with 3812 additions and 6864 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/anyimage/operators/__init__.py
View File

@@ -1,17 +1,9 @@
from .remove_background import RemoveImageBackground, RunBackgroundRemoval
from .cutout_tool import (
CutoutSelectionToShape,
SelectCutoutSelection,
)
from .frame_tool import FrameImages
from .cutout_tool import CLASSES as CUTOUT_CLASSES
from .frame_tool import CLASSES as FRAME_CLASSES
from .mask_tool import EditImageAlpha
from .rectify_tool import RectifyImagePerspective
from .convert_to_plane import (
ConvertToDepthPlane,
ConvertToReliefPlane,
ConvertToPlane,
GenerateDepthPlane,
)
from .rectify_tool import CLASSES as RECTIFY_CLASSES
from .convert_to_plane import CLASSES as PLANE_CLASSES
from .ai_setup import (
ClearModels,
DownloadModel,
@@ -25,21 +17,17 @@ from .convert_to_panorama import CLASSES as PANORAMA_CLASSES
CLASSES = (
*PANORAMA_CLASSES,
*PLANE_CLASSES,
*CUTOUT_CLASSES,
SetupAIEnvironment,
OpenAIEnvironmentSettings,
DownloadModel,
DownloadRequiredModels,
GenerateDepthPlane,
CutoutSelectionToShape,
SelectCutoutSelection,
RunBackgroundRemoval,
ClearModels,
FrameImages,
*FRAME_CLASSES,
EditImageAlpha,
RectifyImagePerspective,
*RECTIFY_CLASSES,
RemoveImageBackground,
UpscaleImage,
ConvertToPlane,
ConvertToDepthPlane,
ConvertToReliefPlane,
)

47
src/anyimage/operators/ai_setup.py
View File

@@ -21,11 +21,8 @@ class SetupAIEnvironment(bpy.types.Operator):
bl_description = "Install the environment and required AI models"
use_mirror: bpy.props.BoolProperty(
name="Use Mirror",
description=(
"Use the China mirrors for uv, Python and Python packages; "
"model download sources are unchanged"
),
name="Use China Mirror",
description="Download Python and packages from China mirrors",
default=False,
)
@@ -34,36 +31,24 @@ class SetupAIEnvironment(bpy.types.Operator):
return not runtime.server_busy()
def invoke(self, context, _event):
return context.window_manager.invoke_props_dialog(self, width=440)
return context.window_manager.invoke_props_dialog(
self, width=440, title="Set Up AI", confirm_text="Install"
)
def draw(self, _context):
layout = self.layout
status = ai_status()
if not status["environment_ready"]:
required_size = sum(
model_download_size(model_key)
for model_key in DEFAULT_MODEL_KEYS
model_download_size(model_key) for model_key in DEFAULT_MODEL_KEYS
)
layout.label(
text="AnyImage AI features run in an isolated Python environment."
)
layout.label(
text="This installs Python, ONNX Runtime and supporting packages."
text="Set up AI features for depth, background removal and upscaling."
)
layout.label(
text=(
"It also downloads MoGe-2 ViT-S Normal, BiRefNet Lite and "
"Real-ESRGAN General WDN x4v3 "
f"({format_download_size(required_size)})."
)
)
layout.label(
text="Allow about 600 MB of downloads and 1 GB of disk space."
)
layout.label(
text=(
"Installation usually takes 5–20 minutes, "
"depending on network speed."
"Downloads the runtime and required models "
f"({format_download_size(required_size)} for models)."
)
)
layout.separator(type="LINE")
@@ -71,7 +56,7 @@ class SetupAIEnvironment(bpy.types.Operator):
return
if status["ready"]:
layout.label(text="The AI Environment is ready.", icon="CHECKMARK")
layout.label(text="AI Server is ready.", icon="CHECKMARK")
return
labels = []
@@ -83,12 +68,13 @@ class SetupAIEnvironment(bpy.types.Operator):
for model_key in status["missing_models"]
)
layout.label(
text="The AI Environment is already installed.",
text="AI environment installed.",
icon="CHECKMARK",
)
layout.label(text=f"Required models to download: {', '.join(labels)}")
layout.label(text=f"Download size: {format_download_size(missing_size)}")
layout.label(text="The download usually takes 2–10 minutes.")
layout.label(text=f"Download: {', '.join(labels)}")
layout.label(
text=f"{format_download_size(missing_size)} · Usually 2–10 minutes."
)
def execute(self, _context):
status = ai_status()
@@ -159,7 +145,8 @@ class DownloadRequiredModels(JobOperatorBase, bpy.types.Operator):
class ClearModels(bpy.types.Operator):
bl_idname = "anyimage.clear_models"
bl_label = "Clear Models"
bl_label = "Unload Models"
bl_description = "Unload AI models to free memory. Downloaded model files are kept."
def execute(self, _context):
try:

3
src/anyimage/operators/clipboard_image/__init__.py
View File

@@ -1,11 +1,12 @@
from .operators import (
CLASSES,
PasteClipboardImage,
TrackNativeCopy,
register_keymaps,
unregister_keymaps,
)
CLASSES = (TrackNativeCopy, PasteClipboardImage)
__all__ = (
"CLASSES",
"PasteClipboardImage",

3
src/anyimage/operators/clipboard_image/operators.py
View File

@@ -148,9 +148,6 @@ class PasteClipboardImage(bpy.types.Operator):
return {"FINISHED"}
CLASSES = (TrackNativeCopy, PasteClipboardImage)
def unregister_keymaps():
global _native_copy_token
for keymap, keymap_item in _keymap_items:

12
src/anyimage/operators/convert_to_panorama/operators.py
View File

@@ -3,7 +3,7 @@
import bpy
from ...common.ai import invoke_ai_setup_if_needed, production_device, report_ai_error, require_environment, require_input_path, require_model
from ...common.image import is_image_empty, video_total_frames
from ...common.image import is_image_empty, is_animated_image
from ...common.color_image import prepare_material_color_input, material_analysis_input, cleanup_material_color_input
from ...preferences import addon_preferences, configured_max_ai_input_size
from ...preferences import configured_geometry_model
@@ -26,7 +26,7 @@ class GeneratePanorama(JobOperatorBase, bpy.types.Operator):
source_object_name: bpy.props.StringProperty(options=_HIDDEN)
source_identity: bpy.props.StringProperty(options=_HIDDEN)
image_identity: bpy.props.StringProperty(options=_HIDDEN)
mesh_detail: bpy.props.IntProperty(default=7, min=0, max=8, options=_HIDDEN)
mesh_detail: bpy.props.IntProperty(default=8, min=0, max=8, options=_HIDDEN)
model: bpy.props.StringProperty(default=DEFAULT_GEOMETRY_MODEL_KEY, options=_HIDDEN)
resolution_level: bpy.props.IntProperty(default=5, min=0, max=9, options=_HIDDEN)
max_input_size: bpy.props.IntProperty(default=DEFAULT_MAX_AI_INPUT_SIZE,
@@ -39,7 +39,7 @@ class GeneratePanorama(JobOperatorBase, bpy.types.Operator):
"input": str(require_input_path(self.input_path)),
"device": production_device().lower(), "model": self.model,
"resolution_level": int(self.resolution_level),
"max_input_size": int(self.max_input_size), "video_frames": 1, "start_frame": 1,
"max_input_size": int(self.max_input_size),
}
def response(self, context, result):
@@ -53,10 +53,10 @@ class GeneratePanorama(JobOperatorBase, bpy.types.Operator):
class ConvertToPanorama(bpy.types.Operator):
bl_idname = "anyimage.convert_to_panorama"
bl_label = "Convert to Panorama"
bl_description = "Convert a static 2:1 equirectangular Image Empty into a radial depth surface"
bl_description = "Turn a still 2:1 panoramic image into a surrounding 3D surface using AI-estimated depth."
bl_options = {"REGISTER", "UNDO"}
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=7, min=0, max=8)
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=8, min=0, max=8)
@classmethod
def poll(cls, context):
@@ -68,7 +68,7 @@ class ConvertToPanorama(bpy.types.Operator):
source = context.object
image = source.data
width, height = image.size
if image.source in {"MOVIE", "SEQUENCE"} or video_total_frames(image) != 1 or height <= 0 or width != height * 2:
if is_animated_image(image) or height <= 0 or width != height * 2:
self.report({"ERROR"}, "Panorama conversion requires a static 2:1 equirectangular image")
return {"CANCELLED"}
setup = invoke_ai_setup_if_needed(self)

3
src/anyimage/operators/convert_to_plane/__init__.py
View File

@@ -7,7 +7,10 @@ from .operators import (
GenerateDepthPlane,
)
CLASSES = (GenerateDepthPlane, ConvertToPlane, ConvertToDepthPlane, ConvertToReliefPlane)
__all__ = (
"CLASSES",
"ConvertToDepthPlane",
"ConvertToReliefPlane",
"ConvertToPlane",

11
src/anyimage/operators/convert_to_plane/operators.py
View File

@@ -75,18 +75,14 @@ class GenerateDepthPlane(JobOperatorBase, bpy.types.Operator):
parameters = {
"input": str(input_path),
"device": production_device().lower(),
"video_frames": 1,
"start_frame": 1,
"max_input_size": int(self.max_input_size),
"plane_type": self.plane_type,
}
parameters.update(
moge2_parameters(
context.scene.anyimage_settings,
input_path,
self.model,
self.resolution_level,
1,
)
)
return parameters
@@ -175,7 +171,10 @@ class ConvertToPlane(bpy.types.Operator):
class ConvertToDepthPlane(bpy.types.Operator):
bl_idname = "anyimage.convert_to_depth_plane"
bl_label = "Convert to Depth Plane"
bl_description = "Convert the complete Image Empty into a camera-projected depth plane"
bl_description = (
"Use AI-estimated depth to turn the image into a 3D surface "
"that follows the source image's perspective."
)
bl_options = {"REGISTER", "UNDO"}
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=6, min=0, max=10)
@@ -193,7 +192,7 @@ class ConvertToDepthPlane(bpy.types.Operator):
class ConvertToReliefPlane(bpy.types.Operator):
bl_idname = "anyimage.convert_to_relief_plane"
bl_label = "Convert to Relief Plane"
bl_description = "Convert the complete Image Empty into a fixed-base relief plane"
bl_description = "Use AI-estimated depth to create a raised relief on a flat base."
bl_options = {"REGISTER", "UNDO"}
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=6, min=0, max=10)

3
src/anyimage/operators/cutout_tool/__init__.py
View File

@@ -1,6 +1,9 @@
from .operators import CutoutSelectionToShape, CutoutTool, SelectCutoutSelection
CLASSES = (CutoutSelectionToShape, SelectCutoutSelection)
__all__ = (
"CLASSES",
"CutoutSelectionToShape",
"CutoutTool",
"SelectCutoutSelection",

396
src/anyimage/operators/cutout_tool/balloon.py Normal file
View File

@@ -0,0 +1,396 @@
"""Compute Poisson and Teddy height profiles for Cutout meshes."""
import math
DEFAULT_INFLATION = 0.45
def distances_to_contour(points, contour):
import numpy as np
points = np.asarray(points, dtype=np.float64)
contour = np.asarray(contour, dtype=np.float64)
squared = np.full(len(points), np.inf, dtype=np.float64)
for start, end in zip(contour, np.roll(contour, -1, axis=0)):
segment = end - start
length_squared = float(segment.dot(segment))
if length_squared <= 1e-16:
continue
factor = np.clip(
(points - start) @ segment / length_squared,
0.0,
1.0,
)
nearest = start + factor[:, None] * segment
squared = np.minimum(
squared,
np.sum((points - nearest) ** 2, axis=1),
)
return np.sqrt(squared)
def chordal_axis(contour, epsilon=1e-9):
import numpy as np
from mathutils import Vector
from mathutils.geometry import delaunay_2d_cdt
contour = np.asarray(contour, dtype=np.float64)
boundary_count = len(contour)
boundary_edges = [
(index, (index + 1) % boundary_count) for index in range(boundary_count)
]
result = delaunay_2d_cdt(
[Vector(point) for point in contour],
boundary_edges,
[list(range(boundary_count))],
1,
epsilon,
)
points = np.asarray(
[tuple(point) for point in result[0]],
dtype=np.float64,
)
triangles = [tuple(face) for face in result[2] if len(face) == 3]
boundary = {tuple(sorted(edge)) for edge in boundary_edges}
node_indices = {}
nodes = []
segments = set()
def edge_node(edge):
key = ("edge", *tuple(sorted(edge)))
if key not in node_indices:
node_indices[key] = len(nodes)
nodes.append((points[edge[0]] + points[edge[1]]) * 0.5)
return node_indices[key]
for face_index, face in enumerate(triangles):
edges = [
(face[0], face[1]),
(face[1], face[2]),
(face[2], face[0]),
]
internal = [edge for edge in edges if tuple(sorted(edge)) not in boundary]
internal_nodes = [edge_node(edge) for edge in internal]
if len(internal_nodes) == 2:
segments.add(tuple(sorted(internal_nodes)))
elif len(internal_nodes) == 3:
key = ("face", face_index)
center_index = len(nodes)
node_indices[key] = center_index
nodes.append(points[list(face)].mean(axis=0))
for midpoint_index in internal_nodes:
segments.add(tuple(sorted((center_index, midpoint_index))))
if not nodes:
center = contour.mean(axis=0)
return (
np.asarray((center,), dtype=np.float64),
[],
np.asarray(
(distances_to_contour((center,), contour)[0],),
dtype=np.float64,
),
)
nodes = np.asarray(nodes, dtype=np.float64)
radii = distances_to_contour(nodes, contour)
active = set(segments)
while len(active) > 1:
degree = [0] * len(nodes)
for start, end in active:
degree[start] += 1
degree[end] += 1
removable = []
for edge in active:
start, end = edge
leaf = start if degree[start] == 1 else end if degree[end] == 1 else None
if leaf is None:
continue
length = float(np.linalg.norm(nodes[start] - nodes[end]))
if length < max(radii[leaf] * 0.5, epsilon):
removable.append(edge)
if not removable:
break
active.difference_update(removable)
if active:
used = sorted({index for edge in active for index in edge})
remap = {old: new for new, old in enumerate(used)}
nodes = nodes[used]
radii = radii[used]
active = {tuple(sorted((remap[start], remap[end]))) for start, end in active}
else:
widest = int(np.argmax(radii))
nodes = nodes[[widest]]
radii = radii[[widest]]
return nodes, sorted(active), radii
def smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
iterations=2,
factor=0.3,
):
import numpy as np
values = np.asarray(heights, dtype=np.float64).copy()
boundary_mask = np.asarray(boundary_mask, dtype=bool)
neighbors = [set() for _index in range(len(values))]
for face in planar_faces:
for start, end in zip(face, face[1:] + face[:1]):
neighbors[start].add(end)
neighbors[end].add(start)
for _iteration in range(iterations):
updated = values.copy()
for index in np.flatnonzero(~boundary_mask):
adjacent = [
neighbor for neighbor in neighbors[index] if not boundary_mask[neighbor]
]
if adjacent:
average = float(values[adjacent].mean())
updated[index] = values[index] * (1.0 - factor) + average * factor
updated[boundary_mask] = 0.0
values = updated
return values
def teddy_balloon_heights(
planar_points,
planar_faces,
contour,
boundary_mask,
inflation,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
boundary_mask = np.asarray(boundary_mask, dtype=bool)
nodes, segments, radii = chordal_axis(contour)
squared_heights = np.zeros(len(points), dtype=np.float64)
for start, end in segments:
axis_start = nodes[start]
axis_end = nodes[end]
axis = axis_end - axis_start
length_squared = float(axis.dot(axis))
if length_squared <= 1e-16:
continue
factors = np.clip(
(points - axis_start) @ axis / length_squared,
0.0,
1.0,
)
centers = axis_start + factors[:, None] * axis
local_radii = radii[start] + factors * (radii[end] - radii[start])
distance_squared = np.sum((points - centers) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
local_radii * local_radii - distance_squared,
)
degree = np.zeros(len(nodes), dtype=np.int64)
for start, end in segments:
degree[start] += 1
degree[end] += 1
for index in np.flatnonzero(degree != 2):
distance_squared = np.sum((points - nodes[index]) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
radii[index] * radii[index] - distance_squared,
)
missing = (~boundary_mask) & (squared_heights <= 0.0)
if np.any(missing):
fallback = distances_to_contour(points[missing], contour)
squared_heights[missing] = fallback * fallback
heights = inflation * np.sqrt(np.maximum(squared_heights, 0.0))
heights[boundary_mask] = 0.0
return smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
)
def cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
tolerance=1e-8,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
faces = np.asarray(planar_faces, dtype=np.int64)
if faces.size == 0:
faces = np.empty((0, 3), dtype=np.int64)
if faces.ndim != 2 or faces.shape[1] != 3:
raise ValueError("Balloon profile faces must be triangles")
boundary_mask = np.asarray(boundary_mask, dtype=bool)
triangles = points[faces]
a, b, c = (triangles[:, index] for index in range(3))
cross = np.abs(
(b[:, 0] - a[:, 0]) * (c[:, 1] - a[:, 1])
- (b[:, 1] - a[:, 1]) * (c[:, 0] - a[:, 0])
)
usable = cross > 1e-14
faces = faces[usable]
a, b, c = a[usable], b[usable], c[usable]
cross = cross[usable]
loads = np.bincount(
faces.ravel(),
weights=np.repeat(cross / 6.0, 3),
minlength=len(points),
)
edges = np.concatenate(
(
faces[:, (1, 2)],
faces[:, (0, 2)],
faces[:, (0, 1)],
),
axis=0,
)
edge_weights = np.concatenate(
(
np.sum((b - a) * (c - a), axis=1),
np.sum((a - b) * (c - b), axis=1),
np.sum((a - c) * (b - c), axis=1),
)
) / np.tile(cross * 2.0, 3)
edges.sort(axis=1)
edges, inverse = np.unique(edges, axis=0, return_inverse=True)
edge_weights = np.bincount(
inverse,
weights=edge_weights,
minlength=len(edges),
)
interior = np.flatnonzero(~boundary_mask)
field = np.zeros(len(points), dtype=np.float64)
if not len(interior):
return field
compact = np.full(len(points), -1, dtype=np.int64)
compact[interior] = np.arange(len(interior), dtype=np.int64)
# Constrained triangulation can create negative cotangent weights.
# Clamping them keeps the Dirichlet system positive definite.
edge_weights = np.maximum(edge_weights, 0.0)
usable = edge_weights > 1e-14
compact_edges = compact[edges[usable]]
edge_weights = edge_weights[usable]
start_compact = compact_edges[:, 0]
end_compact = compact_edges[:, 1]
start_usable = start_compact >= 0
end_usable = end_compact >= 0
diagonal = np.zeros(len(interior), dtype=np.float64)
diagonal += np.bincount(
start_compact[start_usable],
weights=edge_weights[start_usable],
minlength=len(interior),
)
diagonal += np.bincount(
end_compact[end_usable],
weights=edge_weights[end_usable],
minlength=len(interior),
)
if np.any(diagonal <= 1e-14):
raise ValueError("Unable to build a stable Balloon profile")
interior_edges = start_usable & end_usable
edge_starts = start_compact[interior_edges]
edge_ends = end_compact[interior_edges]
weights = edge_weights[interior_edges]
directed_starts = np.concatenate((edge_starts, edge_ends))
directed_ends = np.concatenate((edge_ends, edge_starts))
directed_weights = np.concatenate((weights, weights))
def multiply(values):
result = diagonal * values
if len(directed_weights):
result -= np.bincount(
directed_starts,
weights=directed_weights * values[directed_ends],
minlength=len(values),
)
return result
right_hand_side = loads[interior]
solution = np.zeros(len(interior), dtype=np.float64)
residual = right_hand_side.copy()
preconditioned = residual / diagonal
direction = preconditioned.copy()
residual_product = float(residual.dot(preconditioned))
target = max(float(np.linalg.norm(right_hand_side)) * tolerance, 1e-12)
max_iterations = max(64, min(len(interior) * 4, 4000))
for _iteration in range(max_iterations):
if float(np.linalg.norm(residual)) <= target:
break
multiplied = multiply(direction)
denominator = float(direction.dot(multiplied))
if denominator <= 0.0 or not math.isfinite(denominator):
raise ValueError("Unable to solve the Balloon profile")
step = residual_product / denominator
solution += step * direction
residual -= step * multiplied
preconditioned = residual / diagonal
next_product = float(residual.dot(preconditioned))
if not math.isfinite(next_product):
raise ValueError("Unable to solve the Balloon profile")
if abs(residual_product) <= 1e-30:
break
direction = preconditioned + next_product / residual_product * direction
residual_product = next_product
field[interior] = np.maximum(solution, 0.0)
return field
def poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation=DEFAULT_INFLATION,
):
import numpy as np
field = cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
)
return 2.0 * float(inflation) * np.sqrt(field)
def balloon_profile(
algorithm,
planar_points,
planar_faces,
boundary_mask,
components,
inflation=DEFAULT_INFLATION,
):
import numpy as np
algorithm = str(algorithm).upper()
if algorithm == "POISSON":
return poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation,
)
if algorithm != "TEDDY":
raise ValueError(f"Unknown Balloon profile algorithm: {algorithm}")
result = np.zeros(len(planar_points), dtype=np.float64)
for start, end, component_faces, outer_contour in components:
result[start:end] = teddy_balloon_heights(
np.asarray(planar_points)[start:end],
component_faces,
outer_contour,
np.asarray(boundary_mask)[start:end],
inflation,
)
return result

169
src/anyimage/operators/cutout_tool/depth_calibration.py Normal file
View File

@@ -0,0 +1,169 @@
"""Calibrate metric reference depth and the Depth Balloon base plane."""
PLANE_VARIANCE_FLOOR_RATIO = 0.025
DEPTH_REFERENCE_PERCENTILE = 95.0
ADAPTIVE_DEPTH_TRANSPARENCY_RATIO = 0.05
DEFAULT_DEPTH_BASELINE = 8.0
DEFAULT_DEPTH_DIRECTION = (0.0, 0.0, 1.0)
def use_adaptive_depth_calibration(selection_transparent_fraction):
"""Return whether the Selection input has enough transparent context."""
return float(selection_transparent_fraction) >= ADAPTIVE_DEPTH_TRANSPARENCY_RATIO
def metric_depth_reference(camera_depth, valid, uniform_scale):
"""Return the shared upper-percentile scene depth for both Depth shapes."""
import numpy as np
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth values and validity mask dimensions do not match")
usable = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
if not usable.any():
raise ValueError("The depth field contains no valid metric depth")
if not uniform_scale > 0.0:
raise ValueError("Depth Uniform Scale must be positive")
reference = np.percentile(
camera_depth[usable],
DEPTH_REFERENCE_PERCENTILE,
)
return float(reference) * float(uniform_scale)
def _sample_valid_bilinear(values, valid, x, y):
import numpy as np
height, width = values.shape
x = np.clip(x, 0.0, max(width - 1, 0))
y = np.clip(y, 0.0, max(height - 1, 0))
x0 = np.floor(x).astype(np.int64)
y0 = np.floor(y).astype(np.int64)
x1 = np.minimum(x0 + 1, width - 1)
y1 = np.minimum(y0 + 1, height - 1)
fx = x - x0
fy = y - y0
samples = np.stack(
(
values[y0, x0],
values[y0, x1],
values[y1, x0],
values[y1, x1],
),
axis=1,
)
weights = np.stack(
(
(1.0 - fx) * (1.0 - fy),
fx * (1.0 - fy),
(1.0 - fx) * fy,
fx * fy,
),
axis=1,
)
sample_valid = np.stack(
(
valid[y0, x0],
valid[y0, x1],
valid[y1, x0],
valid[y1, x1],
),
axis=1,
)
weights *= sample_valid
totals = weights.sum(axis=1)
result = np.zeros(len(x), dtype=np.float32)
usable = totals > 1e-8
result[usable] = (samples[usable] * weights[usable]).sum(axis=1) / totals[usable]
return result, usable
def _weighted_plane(coordinates, values, weights):
import numpy as np
total = float(np.sum(weights))
if total <= 1e-12:
return np.zeros(3, dtype=np.float32)
center = np.sum(coordinates * weights[:, None], axis=0) / total
value_center = float(np.sum(values * weights) / total)
centered = coordinates - center
covariance = (centered * weights[:, None]).T @ centered / total
cross = centered.T @ (weights * (values - value_center)) / total
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
largest = max(float(eigenvalues[-1]), 0.0)
if largest <= 1e-12:
slopes = np.zeros(2, dtype=np.float64)
else:
supported = np.maximum(eigenvalues, largest * PLANE_VARIANCE_FLOOR_RATIO)
slopes = eigenvectors @ ((eigenvectors.T @ cross) / supported)
intercept = value_center - float(center @ slopes)
return np.asarray((*slopes, intercept), dtype=np.float32)
def _stable_plane(coordinates, values):
import numpy as np
coordinates = np.asarray(coordinates, dtype=np.float64)
values = np.asarray(values, dtype=np.float64)
if len(coordinates) < 2:
return np.asarray((0.0, 0.0, np.median(values)), dtype=np.float32)
weights = np.ones(len(values), dtype=np.float64)
plane = np.zeros(3, dtype=np.float32)
for _iteration in range(4):
plane = _weighted_plane(coordinates, values, weights)
residual = values - coordinates @ plane[:2] - plane[2]
centered = residual - np.median(residual)
scale = max(float(np.median(np.abs(centered))) * 1.4826, 1e-6)
weights = np.minimum(
1.0,
(2.5 * scale) / np.maximum(np.abs(centered), 1e-12),
)
return plane
def depth_balloon_projection_parameters(
vertices,
vertex_uv,
camera_depth,
valid,
uniform_scale,
):
"""Fit a metric base plane and return its center depth and direction."""
import numpy as np
vertices = np.asarray(vertices, dtype=np.float32)
uv = np.asarray(vertex_uv, dtype=np.float32)
if vertices.ndim != 2 or vertices.shape[1] != 3:
raise ValueError("Depth Balloon vertices must have three components")
if uv.shape != (len(vertices), 2):
raise ValueError("Depth Balloon UV coordinates do not match its vertices")
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth Balloon depth and mask dimensions do not match")
if not uniform_scale > 0.0:
raise ValueError("Depth Balloon Uniform Scale must be positive")
height, width = camera_depth.shape
sample_x = uv[:, 0] * max(width - 1, 0)
sample_y = (1.0 - uv[:, 1]) * max(height - 1, 0)
calibration_valid = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
sampled, usable = _sample_valid_bilinear(
camera_depth,
calibration_valid,
sample_x,
sample_y,
)
if not usable.any():
raise ValueError("MoGe-2 found no valid Depth Balloon samples")
local_coordinates = vertices[:, :2]
metric_depth = sampled[usable] * float(uniform_scale)
plane = _stable_plane(local_coordinates[usable], metric_depth)
reference = metric_depth_reference(
camera_depth,
calibration_valid,
uniform_scale,
)
direction = np.asarray((plane[0], plane[1], 1.0), dtype=np.float64)
direction /= np.linalg.norm(direction)
return reference, tuple(float(value) for value in direction)

565
src/anyimage/operators/cutout_tool/geometry.py
View File

@@ -5,6 +5,7 @@ from dataclasses import dataclass
from ...preferences import addon_preferences
from ...common.image import image_empty_bounds
from . import balloon
@dataclass(frozen=True)
@@ -661,568 +662,6 @@ def triangulate_selection_contours(
return (*result, components) if return_components else result
DEFAULT_INFLATION = 0.45
def distances_to_contour(points, contour):
import numpy as np
points = np.asarray(points, dtype=np.float64)
contour = np.asarray(contour, dtype=np.float64)
squared = np.full(len(points), np.inf, dtype=np.float64)
for start, end in zip(contour, np.roll(contour, -1, axis=0)):
segment = end - start
length_squared = float(segment.dot(segment))
if length_squared <= 1e-16:
continue
factor = np.clip(
(points - start) @ segment / length_squared,
0.0,
1.0,
)
nearest = start + factor[:, None] * segment
squared = np.minimum(
squared,
np.sum((points - nearest) ** 2, axis=1),
)
return np.sqrt(squared)
def chordal_axis(contour, epsilon=1e-9):
import numpy as np
from mathutils import Vector
from mathutils.geometry import delaunay_2d_cdt
contour = np.asarray(contour, dtype=np.float64)
boundary_count = len(contour)
boundary_edges = [
(index, (index + 1) % boundary_count) for index in range(boundary_count)
]
result = delaunay_2d_cdt(
[Vector(point) for point in contour],
boundary_edges,
[list(range(boundary_count))],
1,
epsilon,
)
points = np.asarray(
[tuple(point) for point in result[0]],
dtype=np.float64,
)
triangles = [tuple(face) for face in result[2] if len(face) == 3]
boundary = {tuple(sorted(edge)) for edge in boundary_edges}
node_indices = {}
nodes = []
segments = set()
def edge_node(edge):
key = ("edge", *tuple(sorted(edge)))
if key not in node_indices:
node_indices[key] = len(nodes)
nodes.append((points[edge[0]] + points[edge[1]]) * 0.5)
return node_indices[key]
for face_index, face in enumerate(triangles):
edges = [
(face[0], face[1]),
(face[1], face[2]),
(face[2], face[0]),
]
internal = [edge for edge in edges if tuple(sorted(edge)) not in boundary]
internal_nodes = [edge_node(edge) for edge in internal]
if len(internal_nodes) == 2:
segments.add(tuple(sorted(internal_nodes)))
elif len(internal_nodes) == 3:
key = ("face", face_index)
center_index = len(nodes)
node_indices[key] = center_index
nodes.append(points[list(face)].mean(axis=0))
for midpoint_index in internal_nodes:
segments.add(tuple(sorted((center_index, midpoint_index))))
if not nodes:
center = contour.mean(axis=0)
return (
np.asarray((center,), dtype=np.float64),
[],
np.asarray(
(distances_to_contour((center,), contour)[0],),
dtype=np.float64,
),
)
nodes = np.asarray(nodes, dtype=np.float64)
radii = distances_to_contour(nodes, contour)
active = set(segments)
while len(active) > 1:
degree = [0] * len(nodes)
for start, end in active:
degree[start] += 1
degree[end] += 1
removable = []
for edge in active:
start, end = edge
leaf = start if degree[start] == 1 else end if degree[end] == 1 else None
if leaf is None:
continue
length = float(np.linalg.norm(nodes[start] - nodes[end]))
if length < max(radii[leaf] * 0.5, epsilon):
removable.append(edge)
if not removable:
break
active.difference_update(removable)
if active:
used = sorted({index for edge in active for index in edge})
remap = {old: new for new, old in enumerate(used)}
nodes = nodes[used]
radii = radii[used]
active = {tuple(sorted((remap[start], remap[end]))) for start, end in active}
else:
widest = int(np.argmax(radii))
nodes = nodes[[widest]]
radii = radii[[widest]]
return nodes, sorted(active), radii
def smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
iterations=2,
factor=0.3,
):
import numpy as np
values = np.asarray(heights, dtype=np.float64).copy()
boundary_mask = np.asarray(boundary_mask, dtype=bool)
neighbors = [set() for _index in range(len(values))]
for face in planar_faces:
for start, end in zip(face, face[1:] + face[:1]):
neighbors[start].add(end)
neighbors[end].add(start)
for _iteration in range(iterations):
updated = values.copy()
for index in np.flatnonzero(~boundary_mask):
adjacent = [
neighbor for neighbor in neighbors[index] if not boundary_mask[neighbor]
]
if adjacent:
average = float(values[adjacent].mean())
updated[index] = values[index] * (1.0 - factor) + average * factor
updated[boundary_mask] = 0.0
values = updated
return values
def teddy_balloon_heights(
planar_points,
planar_faces,
contour,
boundary_mask,
inflation,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
boundary_mask = np.asarray(boundary_mask, dtype=bool)
nodes, segments, radii = chordal_axis(contour)
squared_heights = np.zeros(len(points), dtype=np.float64)
for start, end in segments:
axis_start = nodes[start]
axis_end = nodes[end]
axis = axis_end - axis_start
length_squared = float(axis.dot(axis))
if length_squared <= 1e-16:
continue
factors = np.clip(
(points - axis_start) @ axis / length_squared,
0.0,
1.0,
)
centers = axis_start + factors[:, None] * axis
local_radii = radii[start] + factors * (radii[end] - radii[start])
distance_squared = np.sum((points - centers) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
local_radii * local_radii - distance_squared,
)
degree = np.zeros(len(nodes), dtype=np.int64)
for start, end in segments:
degree[start] += 1
degree[end] += 1
for index in np.flatnonzero(degree != 2):
distance_squared = np.sum((points - nodes[index]) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
radii[index] * radii[index] - distance_squared,
)
missing = (~boundary_mask) & (squared_heights <= 0.0)
if np.any(missing):
fallback = distances_to_contour(points[missing], contour)
squared_heights[missing] = fallback * fallback
heights = inflation * np.sqrt(np.maximum(squared_heights, 0.0))
heights[boundary_mask] = 0.0
return smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
)
def cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
tolerance=1e-8,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
faces = np.asarray(planar_faces, dtype=np.int64)
if faces.size == 0:
faces = np.empty((0, 3), dtype=np.int64)
if faces.ndim != 2 or faces.shape[1] != 3:
raise ValueError("Balloon profile faces must be triangles")
boundary_mask = np.asarray(boundary_mask, dtype=bool)
triangles = points[faces]
a, b, c = (triangles[:, index] for index in range(3))
cross = np.abs(
(b[:, 0] - a[:, 0]) * (c[:, 1] - a[:, 1])
- (b[:, 1] - a[:, 1]) * (c[:, 0] - a[:, 0])
)
usable = cross > 1e-14
faces = faces[usable]
a, b, c = a[usable], b[usable], c[usable]
cross = cross[usable]
loads = np.bincount(
faces.ravel(),
weights=np.repeat(cross / 6.0, 3),
minlength=len(points),
)
edges = np.concatenate(
(
faces[:, (1, 2)],
faces[:, (0, 2)],
faces[:, (0, 1)],
),
axis=0,
)
edge_weights = np.concatenate(
(
np.sum((b - a) * (c - a), axis=1),
np.sum((a - b) * (c - b), axis=1),
np.sum((a - c) * (b - c), axis=1),
)
) / np.tile(cross * 2.0, 3)
edges.sort(axis=1)
edges, inverse = np.unique(edges, axis=0, return_inverse=True)
edge_weights = np.bincount(
inverse,
weights=edge_weights,
minlength=len(edges),
)
interior = np.flatnonzero(~boundary_mask)
field = np.zeros(len(points), dtype=np.float64)
if not len(interior):
return field
compact = np.full(len(points), -1, dtype=np.int64)
compact[interior] = np.arange(len(interior), dtype=np.int64)
# Constrained triangulation can create negative cotangent weights.
# Clamping them keeps the Dirichlet system positive definite.
edge_weights = np.maximum(edge_weights, 0.0)
usable = edge_weights > 1e-14
compact_edges = compact[edges[usable]]
edge_weights = edge_weights[usable]
start_compact = compact_edges[:, 0]
end_compact = compact_edges[:, 1]
start_usable = start_compact >= 0
end_usable = end_compact >= 0
diagonal = np.zeros(len(interior), dtype=np.float64)
diagonal += np.bincount(
start_compact[start_usable],
weights=edge_weights[start_usable],
minlength=len(interior),
)
diagonal += np.bincount(
end_compact[end_usable],
weights=edge_weights[end_usable],
minlength=len(interior),
)
if np.any(diagonal <= 1e-14):
raise ValueError("Unable to build a stable Balloon profile")
interior_edges = start_usable & end_usable
edge_starts = start_compact[interior_edges]
edge_ends = end_compact[interior_edges]
weights = edge_weights[interior_edges]
directed_starts = np.concatenate((edge_starts, edge_ends))
directed_ends = np.concatenate((edge_ends, edge_starts))
directed_weights = np.concatenate((weights, weights))
def multiply(values):
result = diagonal * values
if len(directed_weights):
result -= np.bincount(
directed_starts,
weights=directed_weights * values[directed_ends],
minlength=len(values),
)
return result
right_hand_side = loads[interior]
solution = np.zeros(len(interior), dtype=np.float64)
residual = right_hand_side.copy()
preconditioned = residual / diagonal
direction = preconditioned.copy()
residual_product = float(residual.dot(preconditioned))
target = max(float(np.linalg.norm(right_hand_side)) * tolerance, 1e-12)
max_iterations = max(64, min(len(interior) * 4, 4000))
for _iteration in range(max_iterations):
if float(np.linalg.norm(residual)) <= target:
break
multiplied = multiply(direction)
denominator = float(direction.dot(multiplied))
if denominator <= 0.0 or not math.isfinite(denominator):
raise ValueError("Unable to solve the Balloon profile")
step = residual_product / denominator
solution += step * direction
residual -= step * multiplied
preconditioned = residual / diagonal
next_product = float(residual.dot(preconditioned))
if not math.isfinite(next_product):
raise ValueError("Unable to solve the Balloon profile")
if abs(residual_product) <= 1e-30:
break
direction = preconditioned + next_product / residual_product * direction
residual_product = next_product
field[interior] = np.maximum(solution, 0.0)
return field
def poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation=DEFAULT_INFLATION,
):
import numpy as np
field = cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
)
return 2.0 * float(inflation) * np.sqrt(field)
def balloon_profile(
algorithm,
planar_points,
planar_faces,
boundary_mask,
components,
inflation=DEFAULT_INFLATION,
):
import numpy as np
algorithm = str(algorithm).upper()
if algorithm == "POISSON":
return poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation,
)
if algorithm != "TEDDY":
raise ValueError(f"Unknown Balloon profile algorithm: {algorithm}")
result = np.zeros(len(planar_points), dtype=np.float64)
for start, end, component_faces, outer_contour in components:
result[start:end] = teddy_balloon_heights(
np.asarray(planar_points)[start:end],
component_faces,
outer_contour,
np.asarray(boundary_mask)[start:end],
inflation,
)
return result
PLANE_VARIANCE_FLOOR_RATIO = 0.025
DEPTH_REFERENCE_PERCENTILE = 95.0
ADAPTIVE_DEPTH_TRANSPARENCY_RATIO = 0.05
DEFAULT_DEPTH_BASELINE = 8.0
DEFAULT_DEPTH_DIRECTION = (0.0, 0.0, 1.0)
def use_adaptive_depth_calibration(selection_transparent_fraction):
"""Return whether the Selection input has enough transparent context."""
return float(selection_transparent_fraction) >= ADAPTIVE_DEPTH_TRANSPARENCY_RATIO
def metric_depth_reference(camera_depth, valid, uniform_scale):
"""Return the shared upper-percentile scene depth for both Depth shapes."""
import numpy as np
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth values and validity mask dimensions do not match")
usable = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
if not usable.any():
raise ValueError("The depth field contains no valid metric depth")
if not uniform_scale > 0.0:
raise ValueError("Depth Uniform Scale must be positive")
reference = np.percentile(
camera_depth[usable],
DEPTH_REFERENCE_PERCENTILE,
)
return float(reference) * float(uniform_scale)
def _sample_valid_bilinear(values, valid, x, y):
import numpy as np
height, width = values.shape
x = np.clip(x, 0.0, max(width - 1, 0))
y = np.clip(y, 0.0, max(height - 1, 0))
x0 = np.floor(x).astype(np.int64)
y0 = np.floor(y).astype(np.int64)
x1 = np.minimum(x0 + 1, width - 1)
y1 = np.minimum(y0 + 1, height - 1)
fx = x - x0
fy = y - y0
samples = np.stack(
(
values[y0, x0],
values[y0, x1],
values[y1, x0],
values[y1, x1],
),
axis=1,
)
weights = np.stack(
(
(1.0 - fx) * (1.0 - fy),
fx * (1.0 - fy),
(1.0 - fx) * fy,
fx * fy,
),
axis=1,
)
sample_valid = np.stack(
(
valid[y0, x0],
valid[y0, x1],
valid[y1, x0],
valid[y1, x1],
),
axis=1,
)
weights *= sample_valid
totals = weights.sum(axis=1)
result = np.zeros(len(x), dtype=np.float32)
usable = totals > 1e-8
result[usable] = (samples[usable] * weights[usable]).sum(axis=1) / totals[usable]
return result, usable
def _weighted_plane(coordinates, values, weights):
import numpy as np
total = float(np.sum(weights))
if total <= 1e-12:
return np.zeros(3, dtype=np.float32)
center = np.sum(coordinates * weights[:, None], axis=0) / total
value_center = float(np.sum(values * weights) / total)
centered = coordinates - center
covariance = (centered * weights[:, None]).T @ centered / total
cross = centered.T @ (weights * (values - value_center)) / total
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
largest = max(float(eigenvalues[-1]), 0.0)
if largest <= 1e-12:
slopes = np.zeros(2, dtype=np.float64)
else:
supported = np.maximum(eigenvalues, largest * PLANE_VARIANCE_FLOOR_RATIO)
slopes = eigenvectors @ ((eigenvectors.T @ cross) / supported)
intercept = value_center - float(center @ slopes)
return np.asarray((*slopes, intercept), dtype=np.float32)
def _stable_plane(coordinates, values):
import numpy as np
coordinates = np.asarray(coordinates, dtype=np.float64)
values = np.asarray(values, dtype=np.float64)
if len(coordinates) < 2:
return np.asarray((0.0, 0.0, np.median(values)), dtype=np.float32)
weights = np.ones(len(values), dtype=np.float64)
plane = np.zeros(3, dtype=np.float32)
for _iteration in range(4):
plane = _weighted_plane(coordinates, values, weights)
residual = values - coordinates @ plane[:2] - plane[2]
centered = residual - np.median(residual)
scale = max(float(np.median(np.abs(centered))) * 1.4826, 1e-6)
weights = np.minimum(
1.0,
(2.5 * scale) / np.maximum(np.abs(centered), 1e-12),
)
return plane
def depth_balloon_projection_parameters(
vertices,
vertex_uv,
camera_depth,
valid,
uniform_scale,
):
"""Fit a metric base plane and return its center depth and direction."""
import numpy as np
vertices = np.asarray(vertices, dtype=np.float32)
uv = np.asarray(vertex_uv, dtype=np.float32)
if vertices.ndim != 2 or vertices.shape[1] != 3:
raise ValueError("Depth Balloon vertices must have three components")
if uv.shape != (len(vertices), 2):
raise ValueError("Depth Balloon UV coordinates do not match its vertices")
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth Balloon depth and mask dimensions do not match")
if not uniform_scale > 0.0:
raise ValueError("Depth Balloon Uniform Scale must be positive")
height, width = camera_depth.shape
sample_x = uv[:, 0] * max(width - 1, 0)
sample_y = (1.0 - uv[:, 1]) * max(height - 1, 0)
calibration_valid = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
sampled, usable = _sample_valid_bilinear(
camera_depth,
calibration_valid,
sample_x,
sample_y,
)
if not usable.any():
raise ValueError("MoGe-2 found no valid Depth Balloon samples")
local_coordinates = vertices[:, :2]
metric_depth = sampled[usable] * float(uniform_scale)
plane = _stable_plane(local_coordinates[usable], metric_depth)
reference = metric_depth_reference(
camera_depth,
calibration_valid,
uniform_scale,
)
direction = np.asarray((plane[0], plane[1], 1.0), dtype=np.float64)
direction /= np.linalg.norm(direction)
return reference, tuple(float(value) for value in direction)
CUTOUT_ALPHA_THRESHOLD = 0.9
CUTOUT_CLEANUP_UI_SCALE = 100.0
@@ -1313,7 +752,7 @@ def cutout_domain(
planar = np.concatenate(refined_points)
planar_faces, components = refined_faces, refined_components
boundary = boundary_vertices_from_faces(len(planar), planar_faces)
profile = balloon_profile(
profile = balloon.balloon_profile(
balloon_algorithm, planar, planar_faces, boundary, components
)
faces = np.asarray(planar_faces, dtype=np.int32)

9
src/anyimage/operators/cutout_tool/interaction.py
View File

@@ -31,9 +31,9 @@ def _pie_button(
selection_path_json,
generate_normal,
point_spacing,
inward_axis,
alpha_threshold,
fine_outline,
gesture,
):
operator = pie.operator(
"anyimage.cutout_selection_to_shape",
@@ -45,9 +45,9 @@ def _pie_button(
operator.shape = shape
operator.generate_normal = generate_normal
operator.point_spacing = point_spacing
operator.inward_axis = inward_axis
operator.alpha_threshold = alpha_threshold
operator.fine_outline = fine_outline
operator.gesture = gesture
def open_shape_pie(
@@ -55,6 +55,8 @@ def open_shape_pie(
event,
source_object,
selection_path,
*,
gesture="LASSO",
):
source_name = source_object.name
selection_path_json = selection_path.to_json()
@@ -64,7 +66,6 @@ def open_shape_pie(
generate_normal = ready and bool(
getattr(settings, "cutout_generate_normal", False)
)
inward_axis = str(getattr(settings, "cutout_inward_axis", "NEGATIVE_Z"))
alpha_threshold = settings.cutout_alpha_threshold
fine_outline = settings.cutout_fine_outline
@@ -81,9 +82,9 @@ def open_shape_pie(
selection_path_json,
generate_normal,
point_spacing,
inward_axis,
alpha_threshold,
fine_outline,
gesture,
)
context.window_manager.popup_menu_pie(

71
src/anyimage/operators/cutout_tool/object.py
View File

@@ -15,23 +15,24 @@ from ...common.object import (
set_modifier_input,
)
from ...common.node import load_node_group
from .geometry import (
CUTOUT_ALPHA_THRESHOLD,
from .depth_calibration import (
DEFAULT_DEPTH_BASELINE,
DEFAULT_DEPTH_DIRECTION,
depth_balloon_projection_parameters,
metric_depth_reference,
use_adaptive_depth_calibration,
)
from .geometry import (
CUTOUT_ALPHA_THRESHOLD,
crop_plane_bounds,
cutout_cleanup_value,
cutout_sampling_spacing,
depth_balloon_projection_parameters,
metric_depth_reference,
selection_calibration,
use_adaptive_depth_calibration,
world_plane_metric,
)
from .shape import (
BALLOON_ATTRIBUTE_NAME,
NORMAL_SCALE_ATTRIBUTE_NAME,
CUTOUT_X_TO_Z,
CUTOUT_NODE_GROUP_NAMES,
DEPTH_CUTOUT_SHAPES,
)
@@ -43,38 +44,11 @@ class NormalMode(StrEnum):
OBJECT = "OBJECT"
class CutoutInwardAxis(StrEnum):
NEGATIVE_Z = "NEGATIVE_Z"
POSITIVE_X = "POSITIVE_X"
def _cutout_world_matrix(source_matrix, crop_center, inward_axis):
def _cutout_world_matrix(source_matrix, crop_center):
from mathutils import Matrix
center_x, center_y = (float(value) for value in crop_center)
matrix = source_matrix @ Matrix.Translation((center_x, center_y, 0.0))
if CutoutInwardAxis(inward_axis) == CutoutInwardAxis.POSITIVE_X:
matrix @= Matrix(CUTOUT_X_TO_Z)
return matrix
def orient_object_normal_image(image, inward_axis):
"""Convert native Z-axis Cutout normals to the selected object axes."""
if CutoutInwardAxis(inward_axis) == CutoutInwardAxis.NEGATIVE_Z:
return image
import numpy as np
pixels = np.empty(int(image.size[0]) * int(image.size[1]) * 4, dtype=np.float32)
image.pixels.foreach_get(pixels)
rgba = pixels.reshape((-1, 4))
normal = rgba[:, :3] * 2.0 - 1.0
rgba[:, :3] = np.column_stack(
(-normal[:, 2], -normal[:, 0], normal[:, 1])
) * 0.5 + 0.5
image.pixels.foreach_set(pixels)
image.update()
image.pack()
return image
return source_matrix @ Matrix.Translation((center_x, center_y, 0.0))
def _assign_uv(mesh, uv):
@@ -99,7 +73,7 @@ def _assign_float_point_attribute(mesh, name, values):
def load_cutout_node_group(shape):
return load_node_group(CUTOUT_NODE_GROUP_NAMES[shape], asset_revision=23 if shape == "DEPTH_SOLID" else 12 if shape == "DEPTH_BALLOON" else 11)
return load_node_group(CUTOUT_NODE_GROUP_NAMES[shape], asset_revision=26 if shape == "DEPTH_SOLID" else 14 if shape == "DEPTH_BALLOON" else 13)
def _connect_cutout_normal_scale(material):
@@ -134,13 +108,12 @@ def create_shape_object(
depth_metadata=None,
normal_image=None,
normal_mode=NormalMode.NONE,
inward_axis=CutoutInwardAxis.NEGATIVE_Z,
color_image,
gesture="LASSO",
):
import numpy as np
normal_mode = NormalMode(normal_mode)
inward_axis = CutoutInwardAxis(inward_axis)
bounds = crop_plane_bounds(
source_object,
content_bounds,
@@ -229,7 +202,6 @@ def create_shape_object(
mesh_object.matrix_world = _cutout_world_matrix(
source_object.matrix_world,
crop_center,
inward_axis,
)
mesh_object["anyimage_mesh_shape"] = shape
node_group = load_cutout_node_group(shape)
@@ -237,10 +209,6 @@ def create_shape_object(
modifier.node_group = node_group
modifier.show_group_selector = False
for name, value in (
(
"Inward Axis",
0 if inward_axis == CutoutInwardAxis.NEGATIVE_Z else 1,
),
("Cleanup Threshold", cleanup_value),
(
"Thickness",
@@ -249,9 +217,24 @@ def create_shape_object(
):
set_modifier_input(
modifier,
modifier_input_identifier(node_group, name),
modifier_input_identifier(
node_group, name,
subtype="NONE" if name == "Thickness" and shape == "DEPTH_SOLID" else None,
),
value,
)
if shape in {"FLAT", "SOLID", "DEPTH_SOLID"} and gesture == "POLYLINE":
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Mode"),
1,
)
if shape == "SOLID":
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Thickness", subtype="DISTANCE"),
0.5,
)
if shape in DEPTH_CUTOUT_SHAPES:
for name, value in (
("Depth Scale", 1.0),

79
src/anyimage/operators/cutout_tool/operators.py
View File

@@ -13,6 +13,7 @@ from ...preferences import (
from ...properties import (
cutout_alpha_threshold_property,
cutout_fine_outline_property,
cutout_gesture_property,
)
from ...runtime import JobOperatorBase
from ...server.model_catalog import DEFAULT_GEOMETRY_MODEL_KEY
@@ -35,7 +36,7 @@ from ...common.image import (
image_rgba,
load_normal_result_image,
require_image_empty,
video_total_frames,
is_animated_image,
)
from ...common.depth import load_depth_metadata, load_depth_result_image
from ...common.color_image import (
@@ -47,7 +48,7 @@ from ...common.color_image import (
)
from ...common.viewport import (
ImageGesture,
image_edit_drag_keymap,
image_edit_point_keymap,
report_image_edit_exception,
)
from ...common.selection import (
@@ -62,10 +63,8 @@ from .geometry import (
)
from .interaction import open_shape_pie
from .object import (
CutoutInwardAxis,
NormalMode,
create_shape_object,
orient_object_normal_image,
)
from .shape import CUTOUT_SHAPES, DEPTH_CUTOUT_SHAPES
@@ -100,8 +99,8 @@ def create_cutout_shape(
depth_metadata=None,
normal_image=None,
normal_mode=NormalMode.NONE,
inward_axis=CutoutInwardAxis.NEGATIVE_Z,
color_image,
gesture="LASSO",
):
base_shape = build_base_shape(
context,
@@ -125,8 +124,8 @@ def create_cutout_shape(
depth_metadata=depth_metadata,
normal_image=normal_image,
normal_mode=normal_mode,
inward_axis=inward_axis,
color_image=color_image,
gesture=gesture,
)
undo_message = f"Convert to {shape.replace('_', ' ').title()}"
if "FINISHED" not in bpy.ops.ed.undo_push(message=undo_message):
@@ -191,6 +190,7 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
selection_path_json: bpy.props.StringProperty(options=_HIDDEN)
selection_bounds_json: bpy.props.StringProperty(options=_HIDDEN)
shape: bpy.props.StringProperty(default="SOLID", options=_HIDDEN)
gesture: bpy.props.StringProperty(default="LASSO", options=_HIDDEN)
point_spacing: bpy.props.FloatProperty(
default=32.0,
min=0.01,
@@ -215,23 +215,13 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
generate_normal: bpy.props.BoolProperty(default=False, options=_HIDDEN)
alpha_threshold: cutout_alpha_threshold_property()
fine_outline: cutout_fine_outline_property()
inward_axis: bpy.props.StringProperty(
default=CutoutInwardAxis.NEGATIVE_Z.value,
options=_HIDDEN,
)
def _needs_generation(self):
return self.shape in DEPTH_CUTOUT_SHAPES or self.generate_normal
def execute(self, context):
try:
self.inward_axis = getattr(
self,
"inward_axis",
CutoutInwardAxis.NEGATIVE_Z.value,
)
source_object = require_image_empty(self.source_object_name)
if video_total_frames(source_object.data) != 1:
if is_animated_image(source_object.data):
raise ValueError("Cutout currently supports still images")
if self.shape not in CUTOUT_SHAPES:
raise ValueError(f"Unsupported Cutout shape: {self.shape}")
@@ -263,8 +253,9 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
create_cutout_shape(
context, source_object, self.shape, self.point_spacing,
content_values, selection_mask.bounds,
inward_axis=self.inward_axis, alpha_threshold=self.alpha_threshold,
alpha_threshold=self.alpha_threshold,
fine_outline=self.fine_outline, color_image=color_image,
gesture=self.gesture,
)
return {"FINISHED"}
setup_result = invoke_ai_setup_if_needed(self)
@@ -317,21 +308,14 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
)
parameters.update(
moge2_parameters(
context.scene.anyimage_settings,
source_path,
self.model,
self.resolution_level,
1,
)
)
return parameters
def response(self, context, result):
inward_axis = getattr(
self,
"inward_axis",
CutoutInwardAxis.NEGATIVE_Z.value,
)
source_object = require_image_empty(self.source_object_name)
source_bounds = tuple(json.loads(self.selection_bounds_json))
color_image, source_alpha, color_bounds = _generated_color_result(
@@ -358,8 +342,6 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
result.file(normal_key),
source_object,
)
if normal_mode == NormalMode.OBJECT:
orient_object_normal_image(normal_image, inward_axis)
depth_metadata = None
if shape_uses_depth(self.shape):
depth_metadata = load_depth_metadata(result.file("depth_metadata"))
@@ -379,10 +361,10 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
depth_metadata=depth_metadata,
normal_image=normal_image,
normal_mode=normal_mode,
inward_axis=inward_axis,
alpha_threshold=self.alpha_threshold,
fine_outline=self.fine_outline,
color_image=color_image,
gesture=self.gesture,
)
except Exception:
if normal_image is not None and normal_image.users == 0:
@@ -401,22 +383,32 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
class SelectCutoutSelection(ImageGesture, bpy.types.Operator):
bl_idname = "anyimage.select_cutout_selection"
bl_label = "Start Cutout Selection"
bl_description = "Lasso an Image Empty selection, then choose its mesh shape"
bl_description = "Select an Image Empty area with a lasso or polyline, then choose its mesh shape"
bl_options = {"INTERNAL"}
active_tool_id = "anyimage.cutout_lasso"
gesture = "LASSO"
resolve_on_press = True
gesture: cutout_gesture_property()
source_object_name: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
source_matrix_data: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
_pie_event = None
_polyline_confirm_type = None
def _read_gesture_settings(self, context):
self.gesture = context.scene.anyimage_settings.cutout_gesture
def _set_status(self, context):
action = (
"click points; click start, double-click, or Enter for shapes; Backspace removes"
if self.gesture == "POLYLINE" else "drag and release for shapes"
)
context.workspace.status_text_set(
"Cutout Lasso: drag and release for shapes; RMB or Esc cancels"
f"Cutout {self.gesture.title()}: {action}; RMB or Esc cancels"
)
def invoke(self, context, event):
self._polyline_confirm_type = None
self._pie_event = event
return super().invoke(context, event)
@@ -424,6 +416,19 @@ class SelectCutoutSelection(ImageGesture, bpy.types.Operator):
self._pie_event = event
return ImageGesture.modal(self, context, event)
def _confirm_polyline(self, context):
if len(self._path) >= 3:
self._polyline_confirm_type = self._pie_event.type
return {"RUNNING_MODAL"}
def _modal_polyline(self, context, event):
if self._polyline_confirm_type is not None:
if event.type == self._polyline_confirm_type and event.value == "RELEASE":
self._polyline_confirm_type = None
return ImageGesture._confirm_polyline(self, context)
return {"RUNNING_MODAL"}
return ImageGesture._modal_polyline(self, context, event)
def _submit_selection_path(self, context, selection_path):
try:
source_object = require_image_empty(self.source_object_name)
@@ -435,6 +440,7 @@ class SelectCutoutSelection(ImageGesture, bpy.types.Operator):
self._pie_event,
source_object,
selection_path,
gesture=self.gesture,
)
return {"FINISHED"}
@@ -444,19 +450,18 @@ class CutoutTool(bpy.types.WorkSpaceTool):
bl_context_mode = "OBJECT"
bl_idname = "anyimage.cutout_lasso"
bl_label = "Cutout"
bl_description = "Lasso an Image Empty and choose a mesh shape"
bl_description = "Select an Image Empty area with a lasso or polyline and choose a mesh shape"
bl_icon = "ops.mesh.primitive_sphere_add_gizmo"
bl_operator = SelectCutoutSelection.bl_idname
bl_keymap = image_edit_drag_keymap(
bl_keymap = image_edit_point_keymap(
SelectCutoutSelection.bl_idname,
deselect_on_empty=False,
)
def draw_settings(context, layout, _tool):
settings = context.scene.anyimage_settings
status = draw_ai_setup(layout)
draw_ai_property(layout, settings, "cutout_generate_normal", status)
layout.prop(settings, "cutout_gesture")
layout.prop(settings, "cutout_mesh_detail")
layout.prop(settings, "cutout_fine_outline")
layout.prop(settings, "cutout_alpha_threshold")
layout.prop(settings, "cutout_inward_axis")
status = draw_ai_setup(layout)
draw_ai_property(layout, settings, "cutout_generate_normal", status)

6
src/anyimage/operators/cutout_tool/shape.py
View File

@@ -2,12 +2,6 @@
BALLOON_ATTRIBUTE_NAME = "o_balloon"
NORMAL_SCALE_ATTRIBUTE_NAME = "o_normal_scale"
CUTOUT_X_TO_Z = (
(0.0, -1.0, 0.0, 0.0),
(0.0, 0.0, 1.0, 0.0),
(-1.0, 0.0, 0.0, 0.0),
(0.0, 0.0, 0.0, 1.0),
)
CUTOUT_Z_TO_X = (
(0.0, 0.0, -1.0, 0.0),
(-1.0, 0.0, 0.0, 0.0),

648
src/anyimage/operators/frame_tool.py
View File

@@ -1,648 +0,0 @@
import math
import bpy
from ..common.image import (
MAX_PROJECTIVE_OUTPUT_PIXELS,
PROJECTIVE_CHUNK_ROWS,
create_image_edit_result,
homography_from_points,
image_empty_bounds,
image_pixels,
is_image_empty,
premultiplied_rgba,
replace_empty_image,
sample_rgba,
transform_homography,
video_total_frames,
)
from ..preferences import configured_max_frame_resolution
from ..common.viewport import (
active_image_edit_source,
active_view3d_tool_id,
draw_dashed_line,
draw_polygon_fill,
drawing_in_region,
image_edit_drag_keymap,
image_edit_poll,
rectangle_screen_path,
report_image_edit_exception,
)
from ..common.selection import ImageEditWarning, clip_polygon_halfplanes
FRAME_TOOL_ID = "anyimage.frame"
PROJECTIVE_RELATIVE_EPSILON = 1e-12
def _matrix_array(matrix):
import numpy as np
return np.asarray(
tuple(tuple(float(value) for value in row) for row in matrix),
dtype=np.float64,
)
def frame_source_projection(
projection_matrix,
view_matrix,
matrix_world,
bounds,
region_size,
*,
perspective,
):
"""Return normalized image-plane transforms for one frozen view."""
import numpy as np
x_min, x_max, y_min, y_max = (float(value) for value in bounds)
region_width, region_height = (float(value) for value in region_size)
if min(x_max - x_min, y_max - y_min, region_width, region_height) <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_plane = np.asarray(
(
(x_max - x_min, 0.0, x_min),
(0.0, y_min - y_max, y_max),
(0.0, 0.0, 0.0),
(0.0, 0.0, 1.0),
),
dtype=np.float64,
)
world = _matrix_array(matrix_world)
source_to_world = world @ source_plane
source_to_view = _matrix_array(view_matrix) @ source_to_world
source_to_clip = _matrix_array(projection_matrix) @ source_to_world
source_to_screen = np.stack(
(
0.5 * (source_to_clip[0] + source_to_clip[3]),
0.5 * (source_to_clip[1] + source_to_clip[3]),
source_to_clip[3],
)
)
scale = float(np.max(np.abs(source_to_screen)))
if not math.isfinite(scale) or scale <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_to_screen /= scale
if not np.isfinite(source_to_screen).all():
raise ValueError("The Image Empty projection is invalid")
condition = float(np.linalg.cond(source_to_screen))
if not math.isfinite(condition) or condition > 1.0 / np.finfo(np.float64).eps:
raise ValueError("The Image Empty is edge-on to the current view")
return {
"bounds": (x_min, x_max, y_min, y_max),
"origin_depth": -float((_matrix_array(view_matrix) @ world)[2, 3]),
"perspective": bool(perspective),
"region_size": (region_width, region_height),
"screen_to_source": np.linalg.inv(source_to_screen),
"source_to_screen": source_to_screen,
"source_to_view": source_to_view,
"source_to_world": source_to_world,
}
def _projective_points(points, matrix):
"""Map points and return finite results plus per-point validity."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
flat = points.reshape((-1, 2))
homogeneous = np.column_stack((flat, np.ones(len(flat), dtype=np.float64)))
mapped = homogeneous @ np.asarray(matrix, dtype=np.float64).T
magnitude = np.max(np.abs(mapped), axis=1)
valid = (
np.isfinite(mapped).all(axis=1)
& (np.abs(mapped[:, 2]) > PROJECTIVE_RELATIVE_EPSILON * magnitude)
)
result = np.full((len(flat), 2), np.nan, dtype=np.float64)
result[valid] = mapped[valid, :2] / mapped[valid, 2, None]
return result.reshape(points.shape), valid.reshape(points.shape[:-1])
def frame_source_coordinates(source, screen_points):
"""Return source pixels, view depth, validity, and normalized coordinates."""
import numpy as np
screen_points = np.asarray(screen_points, dtype=np.float64)
region_width, region_height = source["region_size"]
normalized_screen = screen_points / (region_width, region_height)
normalized_source, valid = _projective_points(
normalized_screen,
source["screen_to_source"],
)
flat_source = normalized_source.reshape((-1, 2))
homogeneous = np.column_stack(
(flat_source, np.ones(len(flat_source), dtype=np.float64))
)
depth = -(homogeneous @ source["source_to_view"][2])
depth = depth.reshape(valid.shape)
valid &= np.isfinite(depth)
if source["perspective"]:
valid &= depth > 0.0
valid &= (
(normalized_source[..., 0] >= 0.0)
& (normalized_source[..., 0] <= 1.0)
& (normalized_source[..., 1] >= 0.0)
& (normalized_source[..., 1] <= 1.0)
)
image_width, image_height = source["image_size"]
source_points = normalized_source * (image_width, image_height)
source_points[~valid] = -1.0
return source_points, depth, valid, normalized_source
def frame_source_overlap(source, frame_quad):
"""Clip the source plane before dividing its homogeneous projection."""
import numpy as np
frame = np.asarray(frame_quad, dtype=np.float64) / source["region_size"]
left, bottom = frame.min(axis=0)
right, top = frame.max(axis=0)
x, y, w = source["source_to_screen"]
boundaries = [x - left * w, right * w - x, y - bottom * w, top * w - y]
if source["perspective"]:
boundaries.insert(0, -source["source_to_view"][2])
polygon = clip_polygon_halfplanes(
((0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0)), boundaries
)
if len(polygon) < 3:
return ()
screen, valid = _projective_points(polygon, source["source_to_screen"])
if not np.all(valid):
return ()
screen *= source["region_size"]
# Center the coordinates to avoid cancellation in a distant small polygon.
centered = screen - screen[0]
area = abs(np.sum(centered[:, 0] * np.roll(centered[:, 1], -1)
- centered[:, 1] * np.roll(centered[:, 0], -1))) * 0.5
return tuple(map(tuple, screen)) if area > 1e-8 else ()
def polygon_centroid(polygon):
"""Return the area centroid of one non-degenerate screen polygon."""
cross_sum = 0.0
x_sum = 0.0
y_sum = 0.0
for first, second in zip(polygon, (*polygon[1:], polygon[0])):
cross = first[0] * second[1] - second[0] * first[1]
cross_sum += cross
x_sum += (first[0] + second[0]) * cross
y_sum += (first[1] + second[1]) * cross
if abs(cross_sum) <= 1e-12:
raise ValueError("Frame does not overlap the active Image Empty")
return x_sum / (3.0 * cross_sum), y_sum / (3.0 * cross_sum)
def _pixel_quad(image_size):
width, height = (float(value) for value in image_size)
if min(width, height) <= 0.0:
raise ValueError("The source image has no readable dimensions")
return ((0.0, 0.0), (width, 0.0), (width, height), (0.0, height))
def frame_output_size(frame_quad, region_size, max_resolution):
"""Choose output resolution from Frame pixels within the current Region."""
frame_width = abs(float(frame_quad[1][0] - frame_quad[0][0]))
frame_height = abs(float(frame_quad[0][1] - frame_quad[3][1]))
region_width, region_height = (float(value) for value in region_size)
maximum = float(max_resolution)
if min(frame_width, frame_height, region_width, region_height, maximum) <= 0.0:
raise ValueError("The Frame output dimensions must be positive")
scale = min(
1.0,
region_width / frame_width,
region_height / frame_height,
maximum / max(frame_width, frame_height),
math.sqrt(
MAX_PROJECTIVE_OUTPUT_PIXELS / max(frame_width * frame_height, 1.0)
),
)
return (
max(2, int(round(frame_width * scale))),
max(2, int(round(frame_height * scale))),
)
def _frame_source(source_object, region, view3d, view_matrix, *, active):
import numpy as np
image_size = tuple(source_object.data.size)
pixels = image_pixels(source_object.data)
width, height = image_size
bounds = image_empty_bounds(source_object)
source = frame_source_projection(
view3d.perspective_matrix,
view_matrix,
source_object.matrix_world,
bounds,
(region.width, region.height),
perspective=view3d.is_perspective,
)
source.update(
{
"active": bool(active),
"image_size": image_size,
"name": source_object.name,
"object": source_object,
"pixels": premultiplied_rgba(pixels, image_size),
"rgba": np.flipud(pixels.reshape((height, width, 4))),
}
)
return source
def _composite_frame_samples(sources, screen_points):
"""Resolve independently sorted samples with an unweighted bottom RGB base."""
import numpy as np
samples, depths, coordinates = [], [], []
for source in sources:
points, depth, valid, _ = frame_source_coordinates(source, screen_points)
sample = sample_rgba(source["pixels"], points)
sample[~valid] = 0
depths.append(np.where(valid, depth, -math.inf))
samples.append(sample)
coordinates.append(points)
samples = np.stack(samples)
depths = np.stack(depths)
order = np.argsort(-depths, axis=0, kind="stable")
bottom = order[0]
covered = np.isfinite(np.take_along_axis(depths, bottom[None], axis=0)[0])
result = np.zeros_like(samples[0])
for index, source in enumerate(sources):
selected = covered & (bottom == index)
if np.any(selected):
result[..., :3][selected] = sample_rgba(
source["rgba"], coordinates[index][selected]
)[..., :3]
for index in range(len(sources)):
layer = np.take_along_axis(samples, order[index][None, ..., None], axis=0)[0]
alpha = layer[..., 3:4]
result[..., 3:4] = alpha + result[..., 3:4] * (1 - alpha)
if index:
result[..., :3] = layer[..., :3] + result[..., :3] * (1 - alpha)
return result, covered
def _frame_boundary_pixels(sources, points, output_to_screen):
"""Conservatively detect projected contour and depth-order crossings."""
import numpy as np
boundary = np.zeros(points.shape[:-1], dtype=bool)
for source in sources:
width, height = source["region_size"]
mapping = source["screen_to_source"] @ np.diag((1 / width, 1 / height, 1)) @ output_to_screen
u, v, w = mapping
lines = [u, v, u - w, v - w, w]
positive = np.ones_like(boundary)
negative = np.ones_like(boundary)
for line in (u, v, w - u, w - v, w):
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
positive &= value + radius >= 0
negative &= value - radius <= 0
source_boundary = np.zeros_like(boundary)
if source["perspective"]:
lines.append(source["source_to_view"][2] @ mapping)
for line in lines:
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
source_boundary |= np.abs(value) < radius - 1e-10 * max(radius, 1e-20)
boundary |= source_boundary & (positive | negative)
if len(sources) > 1:
first_order = None
for offset in ((-0.499999, -0.499999), (0.499999, -0.499999),
(-0.499999, 0.499999), (0.499999, 0.499999)):
screen = transform_homography(points + offset, output_to_screen)
depths = []
for source in sources:
_, depth, valid, _ = frame_source_coordinates(source, screen)
depths.append(np.where(valid, depth, -math.inf))
order = np.argsort(-np.stack(depths), axis=0, kind="stable")
if first_order is None:
first_order = order
else:
boundary |= np.any(order != first_order, axis=0)
return boundary
def composite_frame_pixels(sources, frame_quad, output_size):
"""Composite projected Image Empty sources into one Blender-order RGBA image."""
import numpy as np
output_width, output_height = (int(value) for value in output_size)
if min(output_width, output_height) <= 0:
raise ValueError("The Frame output dimensions must be positive")
sources = tuple(
sorted(sources, key=lambda source: (source["active"], source["name"]))
)
if not sources or not sources[-1]["active"]:
raise ValueError("Frame requires one active Image Empty")
output_quad = _pixel_quad(output_size)
output_to_screen = homography_from_points(output_quad, frame_quad)
output = np.zeros((output_height, output_width, 4), dtype=np.float32)
covered = np.zeros((output_height, output_width), dtype=bool)
target_x = np.arange(output_width, dtype=np.float64) + 0.5
for row_start in range(0, output_height, PROJECTIVE_CHUNK_ROWS):
row_end = min(row_start + PROJECTIVE_CHUNK_ROWS, output_height)
target_y = np.arange(row_start, row_end, dtype=np.float64) + 0.5
x, y = np.meshgrid(target_x, target_y)
points = np.stack((x, y), axis=-1)
composite, coverage = _composite_frame_samples(
sources, transform_homography(points, output_to_screen)
)
boundary = _frame_boundary_pixels(sources, points, output_to_screen)
if np.any(boundary):
edge_points = points[boundary]
accumulated = np.zeros((len(edge_points), 4), dtype=np.float32)
counts = np.zeros(len(edge_points), dtype=np.int32)
for dy in (-0.375, -0.125, 0.125, 0.375):
for dx in (-0.375, -0.125, 0.125, 0.375):
sample, hit = _composite_frame_samples(
sources,
transform_homography(edge_points + (dx, dy), output_to_screen),
)
accumulated += sample
counts += hit
accumulated[..., 3] /= 16
accumulated[..., :3] /= np.maximum(counts, 1)[..., None]
composite[boundary] = accumulated
coverage[boundary] = counts > 0
covered[row_start:row_end] = coverage
output[row_start:row_end] = composite
output[..., 3][output[..., 3] <= 1e-8] = 0.0
output[~covered] = 0.0
return np.flipud(output).ravel()
def view_facing_frame_geometry(corners, output_size):
"""Return a world matrix array and display size for frame-plane corners."""
import numpy as np
corners = np.asarray(corners, dtype=np.float64)
if corners.shape != (4, 3) or not np.isfinite(corners).all():
raise ValueError("The Frame view plane is invalid")
top_left, top_right, _bottom_right, bottom_left = corners
right = top_right - top_left
up = top_left - bottom_left
width = float(np.linalg.norm(right))
height = float(np.linalg.norm(up))
if min(width, height) <= 1e-10:
raise ValueError("Draw a larger Frame")
right /= width
up /= height
forward = np.cross(right, up)
forward_length = float(np.linalg.norm(forward))
if forward_length <= 1e-10:
raise ValueError("The Frame view plane is invalid")
forward /= forward_length
matrix = np.identity(4, dtype=np.float64)
matrix[:3, 0] = right
matrix[:3, 1] = up
matrix[:3, 2] = forward
matrix[:3, 3] = corners.mean(axis=0)
display_size = width if output_size[0] >= output_size[1] else height
return matrix, display_size
def frame_result_transform(frame_quad, source_origin, region, view3d, output_size):
"""Return the view-facing object matrix and Image Empty display size."""
from mathutils import Matrix
from bpy_extras import view3d_utils
corners = tuple(
tuple(
float(value)
for value in view3d_utils.region_2d_to_location_3d(
region,
view3d,
point,
source_origin,
)
)
for point in frame_quad
)
matrix, display_size = view_facing_frame_geometry(corners, output_size)
return Matrix(tuple(tuple(row) for row in matrix)), display_size
def frame_depth_location(active_source, overlap_polygon):
"""Return the world point whose view depth places the Frame result."""
import numpy as np
source_object = active_source["object"]
if not active_source["perspective"] or active_source["origin_depth"] > 0.0:
return source_object.matrix_world.translation.copy()
centroid = np.asarray((polygon_centroid(overlap_polygon),), dtype=np.float64)
_points, depth, valid, normalized_source = frame_source_coordinates(
active_source,
centroid,
)
if not bool(valid[0]) or depth[0] <= 0.0:
raise ValueError("The Frame result depth is invalid")
source_point = np.asarray(
(normalized_source[0, 0], normalized_source[0, 1], 1.0),
dtype=np.float64,
)
world = active_source["source_to_world"] @ source_point
if not np.isfinite(world).all() or abs(world[3]) <= 1e-12:
raise ValueError("The Frame result depth is invalid")
return tuple(float(value) for value in world[:3] / world[3])
def selected_frame_sources(context):
"""Return selected still Image Empties with the active object last."""
active = active_image_edit_source(context)
sources = [
source for source in context.selected_objects if is_image_empty(source)
]
for source in sources:
if video_total_frames(source.data) > 1:
raise ValueError("Frame only supports still images")
return tuple(
sorted(sources, key=lambda source: (source is active, source.name))
)
def apply_frame(context, source_objects, frame_quad):
"""Bake selected projections into the active Image Empty."""
active = context.active_object
if active not in source_objects:
raise ValueError("The active Image Empty is not a Frame source")
view3d = context.region_data
sources = tuple(
_frame_source(
source,
context.region,
view3d,
view3d.view_matrix,
active=source is active,
)
for source in source_objects
)
active_source = next(source for source in sources if source["active"])
active_overlap = frame_source_overlap(active_source, frame_quad)
if not active_overlap:
raise ImageEditWarning("Frame does not overlap the active Image Empty")
output_size = frame_output_size(
frame_quad,
(context.region.width, context.region.height),
configured_max_frame_resolution(context),
)
pixels = composite_frame_pixels(sources, frame_quad, output_size)
depth_location = frame_depth_location(active_source, active_overlap)
matrix, display_size = frame_result_transform(
frame_quad,
depth_location,
context.region,
view3d,
output_size,
)
result_image = create_image_edit_result(active.data, pixels, output_size)
old_matrix = active.matrix_world.copy()
old_display_size = active.empty_display_size
old_offset = tuple(active.empty_image_offset)
source_images = []
for source in source_objects:
if all(image is not source.data for image in source_images):
source_images.append(source.data)
try:
active.matrix_world = matrix
active.empty_display_size = display_size
active.empty_image_offset = (-0.5, -0.5)
replace_empty_image(
active, result_image,
removed_objects=tuple(source for source in source_objects if source != active),
)
except Exception:
active.matrix_world = old_matrix
active.empty_display_size = old_display_size
active.empty_image_offset = old_offset
raise
for source in source_objects:
if source is not active:
bpy.data.objects.remove(source, do_unlink=True)
for image in source_images:
try:
if image.users == 0 and not image.use_fake_user:
bpy.data.images.remove(image, do_unlink=True)
except (ReferenceError, RuntimeError):
pass
return active
class FrameImages(bpy.types.Operator):
bl_idname = "anyimage.frame_images"
bl_label = "Frame Images"
bl_description = "Bake selected Image Empties into a view-facing frame"
bl_options = {"UNDO"}
_active_name = ""
_source_names = ()
_start = None
_cursor = None
_handle = None
_area_pointer = 0
_region_pointer = 0
@classmethod
def poll(cls, context):
return image_edit_poll(context)
def _draw_overlay(self):
if not drawing_in_region(self._area_pointer, self._region_pointer):
return
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
return
draw_polygon_fill(frame)
draw_dashed_line((*frame, frame[0]))
def _finish(self, context):
if self._handle is not None:
bpy.types.SpaceView3D.draw_handler_remove(self._handle, "WINDOW")
self._handle = None
context.workspace.status_text_set(None)
if context.area is not None:
context.area.tag_redraw()
def cancel(self, context):
self._finish(context)
def invoke(self, context, event):
try:
sources = selected_frame_sources(context)
except (TypeError, ValueError) as error:
report_image_edit_exception(self, error)
return {"CANCELLED"}
self._active_name = context.active_object.name
self._source_names = tuple(source.name for source in sources)
point = (float(event.mouse_region_x), float(event.mouse_region_y))
self._start = point
self._cursor = point
self._area_pointer = context.area.as_pointer()
self._region_pointer = context.region.as_pointer()
self._handle = bpy.types.SpaceView3D.draw_handler_add(
self._draw_overlay, (), "WINDOW", "POST_PIXEL"
)
context.workspace.status_text_set(
"Frame: drag a rectangle to merge selected Image Empties; "
"RMB or Esc cancels"
)
context.window_manager.modal_handler_add(self)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
def modal(self, context, event):
active_tool = active_view3d_tool_id(context)
if active_tool is not None and active_tool != FRAME_TOOL_ID:
self._finish(context)
return {"CANCELLED"}
if event.type in {"ESC", "RIGHTMOUSE"} and event.value == "PRESS":
self._finish(context)
return {"CANCELLED"}
if event.type == "MOUSEMOVE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
if event.type == "LEFTMOUSE" and event.value == "RELEASE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
self._finish(context)
return {"CANCELLED"}
try:
active = bpy.data.objects.get(self._active_name)
sources = tuple(
bpy.data.objects.get(name) for name in self._source_names
)
if (
not is_image_empty(active)
or any(not is_image_empty(source) for source in sources)
or context.active_object is not active
):
raise ValueError("The selected Image Empties changed during Frame")
apply_frame(context, sources, frame)
except (RuntimeError, TypeError, ValueError) as error:
report_image_edit_exception(self, error)
self._finish(context)
return {"CANCELLED"}
self._finish(context)
return {"FINISHED"}
return {"PASS_THROUGH"}
class FrameTool(bpy.types.WorkSpaceTool):
bl_space_type = "VIEW_3D"
bl_context_mode = "OBJECT"
bl_idname = FRAME_TOOL_ID
bl_label = "Frame"
bl_description = "Bake selected Image Empties into a view-facing frame"
bl_icon = "ops.sculpt.border_mask"
bl_operator = FrameImages.bl_idname
bl_keymap = image_edit_drag_keymap(FrameImages.bl_idname)

7
src/anyimage/operators/frame_tool/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
"""Frame conversion operators and workspace tool."""
from .operators import FrameImages, FrameTool
CLASSES = (FrameImages,)
__all__ = ("CLASSES", "FrameImages", "FrameTool")

141
src/anyimage/operators/frame_tool/compositing.py Normal file
View File

@@ -0,0 +1,141 @@
"""Composite projected image samples with independent RGB and Alpha coverage."""
import math
from ...common.image import (
PROJECTIVE_CHUNK_ROWS,
homography_from_points,
sample_rgba,
transform_homography,
)
from .projection import frame_source_coordinates
def _pixel_quad(image_size):
width, height = (float(value) for value in image_size)
if min(width, height) <= 0.0:
raise ValueError("The source image has no readable dimensions")
return ((0.0, 0.0), (width, 0.0), (width, height), (0.0, height))
def _composite_frame_samples(sources, screen_points):
"""Resolve independently sorted samples with an unweighted bottom RGB base."""
import numpy as np
samples, depths, coordinates = [], [], []
for source in sources:
points, depth, valid, _ = frame_source_coordinates(source, screen_points)
sample = sample_rgba(source["pixels"], points)
sample[~valid] = 0
depths.append(np.where(valid, depth, -math.inf))
samples.append(sample)
coordinates.append(points)
samples = np.stack(samples)
depths = np.stack(depths)
order = np.argsort(-depths, axis=0, kind="stable")
bottom = order[0]
covered = np.isfinite(np.take_along_axis(depths, bottom[None], axis=0)[0])
result = np.zeros_like(samples[0])
for index, source in enumerate(sources):
selected = covered & (bottom == index)
if np.any(selected):
result[..., :3][selected] = sample_rgba(
source["rgba"], coordinates[index][selected]
)[..., :3]
for index in range(len(sources)):
layer = np.take_along_axis(samples, order[index][None, ..., None], axis=0)[0]
alpha = layer[..., 3:4]
result[..., 3:4] = alpha + result[..., 3:4] * (1 - alpha)
if index:
result[..., :3] = layer[..., :3] + result[..., :3] * (1 - alpha)
return result, covered
def _frame_boundary_pixels(sources, points, output_to_screen):
"""Conservatively detect projected contour and depth-order crossings."""
import numpy as np
boundary = np.zeros(points.shape[:-1], dtype=bool)
for source in sources:
width, height = source["region_size"]
mapping = source["screen_to_source"] @ np.diag((1 / width, 1 / height, 1)) @ output_to_screen
u, v, w = mapping
lines = [u, v, u - w, v - w, w]
positive = np.ones_like(boundary)
negative = np.ones_like(boundary)
for line in (u, v, w - u, w - v, w):
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
positive &= value + radius >= 0
negative &= value - radius <= 0
source_boundary = np.zeros_like(boundary)
if source["perspective"]:
lines.append(source["source_to_view"][2] @ mapping)
for line in lines:
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
source_boundary |= np.abs(value) < radius - 1e-10 * max(radius, 1e-20)
boundary |= source_boundary & (positive | negative)
if len(sources) > 1:
first_order = None
for offset in ((-0.499999, -0.499999), (0.499999, -0.499999),
(-0.499999, 0.499999), (0.499999, 0.499999)):
screen = transform_homography(points + offset, output_to_screen)
depths = []
for source in sources:
_, depth, valid, _ = frame_source_coordinates(source, screen)
depths.append(np.where(valid, depth, -math.inf))
order = np.argsort(-np.stack(depths), axis=0, kind="stable")
if first_order is None:
first_order = order
else:
boundary |= np.any(order != first_order, axis=0)
return boundary
def composite_frame_pixels(sources, frame_quad, output_size):
"""Composite projected Image Empty sources into one Blender-order RGBA image."""
import numpy as np
output_width, output_height = (int(value) for value in output_size)
if min(output_width, output_height) <= 0:
raise ValueError("The Frame output dimensions must be positive")
sources = tuple(
sorted(sources, key=lambda source: (source["active"], source["name"]))
)
if not sources or not sources[-1]["active"]:
raise ValueError("Frame requires one active Image Empty")
output_quad = _pixel_quad(output_size)
output_to_screen = homography_from_points(output_quad, frame_quad)
output = np.zeros((output_height, output_width, 4), dtype=np.float32)
covered = np.zeros((output_height, output_width), dtype=bool)
target_x = np.arange(output_width, dtype=np.float64) + 0.5
for row_start in range(0, output_height, PROJECTIVE_CHUNK_ROWS):
row_end = min(row_start + PROJECTIVE_CHUNK_ROWS, output_height)
target_y = np.arange(row_start, row_end, dtype=np.float64) + 0.5
x, y = np.meshgrid(target_x, target_y)
points = np.stack((x, y), axis=-1)
composite, coverage = _composite_frame_samples(
sources, transform_homography(points, output_to_screen)
)
boundary = _frame_boundary_pixels(sources, points, output_to_screen)
if np.any(boundary):
edge_points = points[boundary]
accumulated = np.zeros((len(edge_points), 4), dtype=np.float32)
counts = np.zeros(len(edge_points), dtype=np.int32)
for dy in (-0.375, -0.125, 0.125, 0.375):
for dx in (-0.375, -0.125, 0.125, 0.375):
sample, hit = _composite_frame_samples(
sources,
transform_homography(edge_points + (dx, dy), output_to_screen),
)
accumulated += sample
counts += hit
accumulated[..., 3] /= 16
accumulated[..., :3] /= np.maximum(counts, 1)[..., None]
composite[boundary] = accumulated
coverage[boundary] = counts > 0
covered[row_start:row_end] = coverage
output[row_start:row_end] = composite
output[..., 3][output[..., 3] <= 1e-8] = 0.0
output[~covered] = 0.0
return np.flipud(output).ravel()

264
src/anyimage/operators/frame_tool/operators.py Normal file
View File

@@ -0,0 +1,264 @@
"""Frame interaction, source preparation, and Blender image replacement."""
import bpy
from ...common.image import (
create_image_edit_result,
image_empty_bounds,
image_pixels,
is_image_empty,
premultiplied_rgba,
replace_empty_image,
is_animated_image,
)
from ...preferences import configured_max_frame_resolution
from ...common.viewport import (
active_image_edit_source,
active_view3d_tool_id,
draw_dashed_line,
draw_polygon_fill,
drawing_in_region,
image_edit_drag_keymap,
image_edit_poll,
rectangle_screen_path,
report_image_edit_exception,
)
from ...common.selection import ImageEditWarning
from .projection import (
frame_depth_location,
frame_output_size,
frame_result_transform,
frame_source_overlap,
frame_source_projection,
)
from .compositing import composite_frame_pixels
FRAME_TOOL_ID = "anyimage.frame"
def _frame_source(source_object, region, view3d, view_matrix, *, active):
import numpy as np
image_size = tuple(source_object.data.size)
pixels = image_pixels(source_object.data)
width, height = image_size
bounds = image_empty_bounds(source_object)
source = frame_source_projection(
view3d.perspective_matrix,
view_matrix,
source_object.matrix_world,
bounds,
(region.width, region.height),
perspective=view3d.is_perspective,
)
source.update(
{
"active": bool(active),
"image_size": image_size,
"name": source_object.name,
"object": source_object,
"pixels": premultiplied_rgba(pixels, image_size),
"rgba": np.flipud(pixels.reshape((height, width, 4))),
}
)
return source
def selected_frame_sources(context):
"""Return selected still Image Empties with the active object last."""
active = active_image_edit_source(context)
sources = [
source for source in context.selected_objects if is_image_empty(source)
]
for source in sources:
if is_animated_image(source.data):
raise ValueError("Frame only supports still images")
return tuple(
sorted(sources, key=lambda source: (source is active, source.name))
)
def apply_frame(context, source_objects, frame_quad):
"""Bake selected projections into the active Image Empty."""
active = context.active_object
if active not in source_objects:
raise ValueError("The active Image Empty is not a Frame source")
view3d = context.region_data
sources = tuple(
_frame_source(
source,
context.region,
view3d,
view3d.view_matrix,
active=source is active,
)
for source in source_objects
)
active_source = next(source for source in sources if source["active"])
active_overlap = frame_source_overlap(active_source, frame_quad)
if not active_overlap:
raise ImageEditWarning("Frame does not overlap the active Image Empty")
output_size = frame_output_size(
frame_quad,
(context.region.width, context.region.height),
configured_max_frame_resolution(context),
)
pixels = composite_frame_pixels(sources, frame_quad, output_size)
depth_location = frame_depth_location(active_source, active_overlap)
matrix, display_size = frame_result_transform(
frame_quad,
depth_location,
context.region,
view3d,
output_size,
)
result_image = create_image_edit_result(active.data, pixels, output_size)
old_matrix = active.matrix_world.copy()
old_display_size = active.empty_display_size
old_offset = tuple(active.empty_image_offset)
source_images = []
for source in source_objects:
if all(image is not source.data for image in source_images):
source_images.append(source.data)
try:
active.matrix_world = matrix
active.empty_display_size = display_size
active.empty_image_offset = (-0.5, -0.5)
replace_empty_image(
active, result_image,
removed_objects=tuple(source for source in source_objects if source != active),
)
except Exception:
active.matrix_world = old_matrix
active.empty_display_size = old_display_size
active.empty_image_offset = old_offset
raise
for source in source_objects:
if source is not active:
bpy.data.objects.remove(source, do_unlink=True)
for image in source_images:
try:
if image.users == 0 and not image.use_fake_user:
bpy.data.images.remove(image, do_unlink=True)
except (ReferenceError, RuntimeError):
pass
return active
class FrameImages(bpy.types.Operator):
bl_idname = "anyimage.frame_images"
bl_label = "Frame Images"
bl_description = "Draw a rectangle to combine selected images into one image from the current view."
bl_options = {"UNDO"}
_active_name = ""
_source_names = ()
_start = None
_cursor = None
_handle = None
_area_pointer = 0
_region_pointer = 0
@classmethod
def poll(cls, context):
return image_edit_poll(context)
def _draw_overlay(self):
if not drawing_in_region(self._area_pointer, self._region_pointer):
return
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
return
draw_polygon_fill(frame)
draw_dashed_line((*frame, frame[0]))
def _finish(self, context):
if self._handle is not None:
bpy.types.SpaceView3D.draw_handler_remove(self._handle, "WINDOW")
self._handle = None
context.workspace.status_text_set(None)
if context.area is not None:
context.area.tag_redraw()
def cancel(self, context):
self._finish(context)
def invoke(self, context, event):
try:
sources = selected_frame_sources(context)
except (TypeError, ValueError) as error:
report_image_edit_exception(self, error)
return {"CANCELLED"}
self._active_name = context.active_object.name
self._source_names = tuple(source.name for source in sources)
point = (float(event.mouse_region_x), float(event.mouse_region_y))
self._start = point
self._cursor = point
self._area_pointer = context.area.as_pointer()
self._region_pointer = context.region.as_pointer()
self._handle = bpy.types.SpaceView3D.draw_handler_add(
self._draw_overlay, (), "WINDOW", "POST_PIXEL"
)
context.workspace.status_text_set(
"Frame: drag a rectangle to merge selected Image Empties; "
"RMB or Esc cancels"
)
context.window_manager.modal_handler_add(self)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
def modal(self, context, event):
active_tool = active_view3d_tool_id(context)
if active_tool is not None and active_tool != FRAME_TOOL_ID:
self._finish(context)
return {"CANCELLED"}
if event.type in {"ESC", "RIGHTMOUSE"} and event.value == "PRESS":
self._finish(context)
return {"CANCELLED"}
if event.type == "MOUSEMOVE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
if event.type == "LEFTMOUSE" and event.value == "RELEASE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
self._finish(context)
return {"CANCELLED"}
try:
active = bpy.data.objects.get(self._active_name)
sources = tuple(
bpy.data.objects.get(name) for name in self._source_names
)
if (
not is_image_empty(active)
or any(not is_image_empty(source) for source in sources)
or context.active_object is not active
):
raise ValueError("The selected Image Empties changed during Frame")
apply_frame(context, sources, frame)
except (RuntimeError, TypeError, ValueError) as error:
report_image_edit_exception(self, error)
self._finish(context)
return {"CANCELLED"}
self._finish(context)
return {"FINISHED"}
return {"PASS_THROUGH"}
class FrameTool(bpy.types.WorkSpaceTool):
bl_space_type = "VIEW_3D"
bl_context_mode = "OBJECT"
bl_idname = FRAME_TOOL_ID
bl_label = "Frame"
bl_description = "Draw a rectangle to combine selected images into one image from the current view."
bl_icon = "ops.sculpt.border_mask"
bl_operator = FrameImages.bl_idname
bl_keymap = image_edit_drag_keymap(FrameImages.bl_idname)

264
src/anyimage/operators/frame_tool/projection.py Normal file
View File

@@ -0,0 +1,264 @@
"""Project image planes and place the view-facing Frame result."""
import math
from ...common.image import MAX_PROJECTIVE_OUTPUT_PIXELS
from ...common.selection import clip_polygon_halfplanes
PROJECTIVE_RELATIVE_EPSILON = 1e-12
def _matrix_array(matrix):
import numpy as np
return np.asarray(
tuple(tuple(float(value) for value in row) for row in matrix),
dtype=np.float64,
)
def frame_source_projection(
projection_matrix,
view_matrix,
matrix_world,
bounds,
region_size,
*,
perspective,
):
"""Return normalized image-plane transforms for one frozen view."""
import numpy as np
x_min, x_max, y_min, y_max = (float(value) for value in bounds)
region_width, region_height = (float(value) for value in region_size)
if min(x_max - x_min, y_max - y_min, region_width, region_height) <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_plane = np.asarray(
(
(x_max - x_min, 0.0, x_min),
(0.0, y_min - y_max, y_max),
(0.0, 0.0, 0.0),
(0.0, 0.0, 1.0),
),
dtype=np.float64,
)
world = _matrix_array(matrix_world)
source_to_world = world @ source_plane
source_to_view = _matrix_array(view_matrix) @ source_to_world
source_to_clip = _matrix_array(projection_matrix) @ source_to_world
source_to_screen = np.stack(
(
0.5 * (source_to_clip[0] + source_to_clip[3]),
0.5 * (source_to_clip[1] + source_to_clip[3]),
source_to_clip[3],
)
)
scale = float(np.max(np.abs(source_to_screen)))
if not math.isfinite(scale) or scale <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_to_screen /= scale
if not np.isfinite(source_to_screen).all():
raise ValueError("The Image Empty projection is invalid")
condition = float(np.linalg.cond(source_to_screen))
if not math.isfinite(condition) or condition > 1.0 / np.finfo(np.float64).eps:
raise ValueError("The Image Empty is edge-on to the current view")
return {
"bounds": (x_min, x_max, y_min, y_max),
"origin_depth": -float((_matrix_array(view_matrix) @ world)[2, 3]),
"perspective": bool(perspective),
"region_size": (region_width, region_height),
"screen_to_source": np.linalg.inv(source_to_screen),
"source_to_screen": source_to_screen,
"source_to_view": source_to_view,
"source_to_world": source_to_world,
}
def _projective_points(points, matrix):
"""Map points and return finite results plus per-point validity."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
flat = points.reshape((-1, 2))
homogeneous = np.column_stack((flat, np.ones(len(flat), dtype=np.float64)))
mapped = homogeneous @ np.asarray(matrix, dtype=np.float64).T
magnitude = np.max(np.abs(mapped), axis=1)
valid = (
np.isfinite(mapped).all(axis=1)
& (np.abs(mapped[:, 2]) > PROJECTIVE_RELATIVE_EPSILON * magnitude)
)
result = np.full((len(flat), 2), np.nan, dtype=np.float64)
result[valid] = mapped[valid, :2] / mapped[valid, 2, None]
return result.reshape(points.shape), valid.reshape(points.shape[:-1])
def frame_source_coordinates(source, screen_points):
"""Return source pixels, view depth, validity, and normalized coordinates."""
import numpy as np
screen_points = np.asarray(screen_points, dtype=np.float64)
region_width, region_height = source["region_size"]
normalized_screen = screen_points / (region_width, region_height)
normalized_source, valid = _projective_points(
normalized_screen,
source["screen_to_source"],
)
flat_source = normalized_source.reshape((-1, 2))
homogeneous = np.column_stack(
(flat_source, np.ones(len(flat_source), dtype=np.float64))
)
depth = -(homogeneous @ source["source_to_view"][2])
depth = depth.reshape(valid.shape)
valid &= np.isfinite(depth)
if source["perspective"]:
valid &= depth > 0.0
valid &= (
(normalized_source[..., 0] >= 0.0)
& (normalized_source[..., 0] <= 1.0)
& (normalized_source[..., 1] >= 0.0)
& (normalized_source[..., 1] <= 1.0)
)
image_width, image_height = source["image_size"]
source_points = normalized_source * (image_width, image_height)
source_points[~valid] = -1.0
return source_points, depth, valid, normalized_source
def frame_source_overlap(source, frame_quad):
"""Clip the source plane before dividing its homogeneous projection."""
import numpy as np
frame = np.asarray(frame_quad, dtype=np.float64) / source["region_size"]
left, bottom = frame.min(axis=0)
right, top = frame.max(axis=0)
x, y, w = source["source_to_screen"]
boundaries = [x - left * w, right * w - x, y - bottom * w, top * w - y]
if source["perspective"]:
boundaries.insert(0, -source["source_to_view"][2])
polygon = clip_polygon_halfplanes(
((0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0)), boundaries
)
if len(polygon) < 3:
return ()
screen, valid = _projective_points(polygon, source["source_to_screen"])
if not np.all(valid):
return ()
screen *= source["region_size"]
# Center the coordinates to avoid cancellation in a distant small polygon.
centered = screen - screen[0]
area = abs(np.sum(centered[:, 0] * np.roll(centered[:, 1], -1)
- centered[:, 1] * np.roll(centered[:, 0], -1))) * 0.5
return tuple(map(tuple, screen)) if area > 1e-8 else ()
def polygon_centroid(polygon):
"""Return the area centroid of one non-degenerate screen polygon."""
cross_sum = 0.0
x_sum = 0.0
y_sum = 0.0
for first, second in zip(polygon, (*polygon[1:], polygon[0])):
cross = first[0] * second[1] - second[0] * first[1]
cross_sum += cross
x_sum += (first[0] + second[0]) * cross
y_sum += (first[1] + second[1]) * cross
if abs(cross_sum) <= 1e-12:
raise ValueError("Frame does not overlap the active Image Empty")
return x_sum / (3.0 * cross_sum), y_sum / (3.0 * cross_sum)
def frame_output_size(frame_quad, region_size, max_resolution):
"""Choose output resolution from Frame pixels within the current Region."""
frame_width = abs(float(frame_quad[1][0] - frame_quad[0][0]))
frame_height = abs(float(frame_quad[0][1] - frame_quad[3][1]))
region_width, region_height = (float(value) for value in region_size)
maximum = float(max_resolution)
if min(frame_width, frame_height, region_width, region_height, maximum) <= 0.0:
raise ValueError("The Frame output dimensions must be positive")
scale = min(
1.0,
region_width / frame_width,
region_height / frame_height,
maximum / max(frame_width, frame_height),
math.sqrt(
MAX_PROJECTIVE_OUTPUT_PIXELS / max(frame_width * frame_height, 1.0)
),
)
return (
max(2, int(round(frame_width * scale))),
max(2, int(round(frame_height * scale))),
)
def view_facing_frame_geometry(corners, output_size):
"""Return a world matrix array and display size for frame-plane corners."""
import numpy as np
corners = np.asarray(corners, dtype=np.float64)
if corners.shape != (4, 3) or not np.isfinite(corners).all():
raise ValueError("The Frame view plane is invalid")
top_left, top_right, _bottom_right, bottom_left = corners
right = top_right - top_left
up = top_left - bottom_left
width = float(np.linalg.norm(right))
height = float(np.linalg.norm(up))
if min(width, height) <= 1e-10:
raise ValueError("Draw a larger Frame")
right /= width
up /= height
forward = np.cross(right, up)
forward_length = float(np.linalg.norm(forward))
if forward_length <= 1e-10:
raise ValueError("The Frame view plane is invalid")
forward /= forward_length
matrix = np.identity(4, dtype=np.float64)
matrix[:3, 0] = right
matrix[:3, 1] = up
matrix[:3, 2] = forward
matrix[:3, 3] = corners.mean(axis=0)
display_size = width if output_size[0] >= output_size[1] else height
return matrix, display_size
def frame_result_transform(frame_quad, source_origin, region, view3d, output_size):
"""Return the view-facing object matrix and Image Empty display size."""
from mathutils import Matrix
from bpy_extras import view3d_utils
corners = tuple(
tuple(
float(value)
for value in view3d_utils.region_2d_to_location_3d(
region,
view3d,
point,
source_origin,
)
)
for point in frame_quad
)
matrix, display_size = view_facing_frame_geometry(corners, output_size)
return Matrix(tuple(tuple(row) for row in matrix)), display_size
def frame_depth_location(active_source, overlap_polygon):
"""Return the world point whose view depth places the Frame result."""
import numpy as np
source_object = active_source["object"]
if not active_source["perspective"] or active_source["origin_depth"] > 0.0:
return source_object.matrix_world.translation.copy()
centroid = np.asarray((polygon_centroid(overlap_polygon),), dtype=np.float64)
_points, depth, valid, normalized_source = frame_source_coordinates(
active_source,
centroid,
)
if not bool(valid[0]) or depth[0] <= 0.0:
raise ValueError("The Frame result depth is invalid")
source_point = np.asarray(
(normalized_source[0, 0], normalized_source[0, 1], 1.0),
dtype=np.float64,
)
world = active_source["source_to_world"] @ source_point
if not np.isfinite(world).all() or abs(world[3]) <= 1e-12:
raise ValueError("The Frame result depth is invalid")
return tuple(float(value) for value in world[:3] / world[3])

6
src/anyimage/operators/mask_tool.py
View File

@@ -39,7 +39,7 @@ def compose_alpha(source_alpha, mask_values, mode):
mode = str(mode)
if mode == "SET":
result = alpha * mask
elif mode == "ADD":
elif mode == "EXTEND":
result = np.maximum(alpha, mask)
elif mode == "SUBTRACT":
result = alpha * (1.0 - mask)
@@ -72,7 +72,7 @@ def rasterize_brush_path(image_size, path, radius, project_path):
class EditImageAlpha(ImageGesture, bpy.types.Operator):
bl_idname = "anyimage.edit_image_alpha"
bl_label = "Edit Image Alpha"
bl_description = "Edit Image Empty Alpha with a lasso, brush, or polyline"
bl_description = "Hide or reveal image areas with a lasso, brush, or polyline."
bl_options = {"UNDO"}
active_tool_id = MASK_TOOL_ID
resolve_on_press = True
@@ -157,7 +157,7 @@ class MaskTool(bpy.types.WorkSpaceTool):
bl_context_mode = "OBJECT"
bl_idname = MASK_TOOL_ID
bl_label = "Mask"
bl_description = "Edit Image Empty Alpha with a lasso, brush, or polyline"
bl_description = "Hide or reveal image areas with a lasso, brush, or polyline."
bl_icon = "ops.sculpt.lasso_mask"
bl_operator = EditImageAlpha.bl_idname
bl_cursor = "PAINT_CROSS"

7
src/anyimage/operators/rectify_tool/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
"""Perspective correction operators and workspace tool."""
from .operators import RectifyImagePerspective, RectifyTool
CLASSES = (RectifyImagePerspective,)
__all__ = ("CLASSES", "RectifyImagePerspective", "RectifyTool")

144
src/anyimage/operators/rectify_tool/geometry.py Normal file
View File

@@ -0,0 +1,144 @@
"""Validate perspective quads and resample the selected image region."""
import math
from ...common.selection import (
clip_polygon_halfplanes,
trim_pixels_to_alpha,
)
from ...common.image import (
projective_output_size,
warp_projective_pixels,
)
MIN_INTERACTIVE_ASPECT = 0.05
MAX_INTERACTIVE_ASPECT = 20.0
def validate_perspective_quad(points, min_area=1e-6):
"""Return four convex corners without changing their correspondence."""
import numpy as np
quad = np.asarray(points, dtype=np.float64)
if quad.shape != (4, 2) or not np.isfinite(quad).all():
raise ValueError("Rectify requires four valid image corners")
edges = np.roll(quad, -1, axis=0) - quad
if np.any(np.linalg.norm(edges, axis=1) < 2.0):
raise ValueError("Rectify corners are too close together")
crosses = np.cross(edges, np.roll(edges, -1, axis=0))
if np.any(np.abs(crosses) <= 1e-8) or not (
np.all(crosses > 0.0) or np.all(crosses < 0.0)
):
raise ValueError("Rectify corners must define one convex quadrilateral")
area = 0.5 * abs(
float(
np.dot(quad[:, 0], np.roll(quad[:, 1], -1))
- np.dot(quad[:, 1], np.roll(quad[:, 0], -1))
)
)
if area < min_area:
raise ValueError("Draw a larger Rectify quad")
return quad
def canonical_perspective_quad(points, min_area=1e-6):
"""Order four unordered points as source-oriented top-left through bottom-left."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
if points.shape != (4, 2) or not np.isfinite(points).all():
raise ValueError("Rectify requires four valid image corners")
distances = np.linalg.norm(points[:, None] - points[None, :], axis=2)
distances += np.eye(4) * 1e9
if float(distances.min()) < 2.0:
raise ValueError("Rectify corners are too close together")
center = points.mean(axis=0)
ring = points[np.argsort(np.arctan2(points[:, 1] - center[1], points[:, 0] - center[0]))]
ring = validate_perspective_quad(ring, min_area)
candidates = []
for winding in (ring, ring[::-1]):
for start in range(4):
candidate = np.roll(winding, -start, axis=0)
top = candidate[1] - candidate[0]
left = candidate[3] - candidate[0]
score = top[0] / np.linalg.norm(top) + left[1] / np.linalg.norm(left)
candidates.append((float(score), -candidate[0, 1], -candidate[0, 0], candidate))
return max(candidates, key=lambda item: item[:3])[3]
def perspective_quad_overlaps_image(points, image_size):
import numpy as np
width, height = image_size
if min(width, height) <= 0:
raise ValueError("The source image has no readable dimensions")
clipped = np.asarray(clip_polygon_halfplanes(
points, ((1, 0, 0), (-1, 0, width), (0, 1, 0), (0, -1, height))
))
if clipped.ndim != 2 or len(clipped) < 3:
return False
area = 0.5 * abs(
float(
np.dot(clipped[:, 0], np.roll(clipped[:, 1], -1))
- np.dot(clipped[:, 1], np.roll(clipped[:, 0], -1))
)
)
return area > 1e-6
def perspective_quad_aspect(quad):
"""Estimate an initial output aspect from the visible quad edge lengths."""
import numpy as np
quad = validate_perspective_quad(quad)
width = 0.5 * (
np.linalg.norm(quad[1] - quad[0]) + np.linalg.norm(quad[2] - quad[3])
)
height = 0.5 * (
np.linalg.norm(quad[2] - quad[1]) + np.linalg.norm(quad[3] - quad[0])
)
return min(
max(float(width / max(height, 1e-8)), MIN_INTERACTIVE_ASPECT),
MAX_INTERACTIVE_ASPECT,
)
def perspective_placement_bounds(quad, output_size, trim_bounds):
"""Map a rectified Alpha trim back to an approximate source-image rectangle."""
quad = validate_perspective_quad(quad)
source_left = math.floor(float(quad[:, 0].min()))
source_top = math.floor(float(quad[:, 1].min()))
source_right = math.ceil(float(quad[:, 0].max()))
source_bottom = math.ceil(float(quad[:, 1].max()))
output_width, output_height = (float(value) for value in output_size)
trim_left, trim_top, trim_right, trim_bottom = trim_bounds
source_width = source_right - source_left
source_height = source_bottom - source_top
return (
source_left + trim_left * source_width / output_width,
source_top + trim_top * source_height / output_height,
source_left + trim_right * source_width / output_width,
source_top + trim_bottom * source_height / output_height,
)
def extract_perspective_pixels(
source_pixels,
source_size,
quad,
aspect_ratio,
):
output_size = projective_output_size(quad, aspect_ratio)
pixels = warp_projective_pixels(
source_pixels, source_size, quad, output_size
)
pixels, trim_bounds = trim_pixels_to_alpha(pixels, (0, 0, *output_size))
trimmed_size = (
trim_bounds[2] - trim_bounds[0],
trim_bounds[3] - trim_bounds[1],
)
placement_bounds = perspective_placement_bounds(
quad, output_size, trim_bounds
)
return pixels, trimmed_size, placement_bounds

326
src/anyimage/operators/rectify_tool.py → src/anyimage/operators/rectify_tool/operators.py
View File

@@ -1,9 +1,11 @@
"""Rectify interaction and Blender image replacement."""
import json
import math
import bpy
from ..common.viewport import (
from ...common.viewport import (
active_view3d_tool_id,
deserialize_matrix,
draw_dashed_line,
@@ -16,311 +18,45 @@ from ..common.viewport import (
screen_path_to_image_pixels,
serialize_matrix,
)
from ..common.selection import clip_polygon_halfplanes, trim_pixels_to_alpha
from ..common.image import (
from ...common.image import (
create_image_edit_result,
image_base_name,
image_pixels,
image_empty_bounds,
projective_output_size,
replace_empty_image,
require_image_empty,
video_total_frames,
is_animated_image,
warp_projective_pixels,
)
MIN_QUAD_AREA = 16.0
RECTIFY_TOOL_ID = "anyimage.rectify"
PREVIEW_TEXTURE_SIZE = 512
ASPECT_PIXELS_PER_DOUBLE = 240.0
MIN_INTERACTIVE_ASPECT = 0.05
MAX_INTERACTIVE_ASPECT = 20.0
ASPECT_PRESETS = (
("9 : 16", 9.0 / 16.0),
("3 : 4", 3.0 / 4.0),
("1 : 1", 1.0),
("4 : 3", 4.0 / 3.0),
("16 : 9", 16.0 / 9.0),
from .geometry import (
MAX_INTERACTIVE_ASPECT,
MIN_INTERACTIVE_ASPECT,
canonical_perspective_quad,
extract_perspective_pixels,
perspective_quad_aspect,
perspective_quad_overlaps_image,
validate_perspective_quad,
)
from .preview import (
PREVIEW_TEXTURE_SIZE,
aspect_ratio_from_mouse,
create_perspective_preview_image,
draw_aspect_hud,
draw_preview_frame,
interactive_preview_polygon,
preview_draw_bounds,
preview_texture_draw_options,
snapped_aspect_ratio,
)
def validate_perspective_quad(points, min_area=1e-6):
"""Return four convex corners without changing their correspondence."""
import numpy as np
quad = np.asarray(points, dtype=np.float64)
if quad.shape != (4, 2) or not np.isfinite(quad).all():
raise ValueError("Rectify requires four valid image corners")
edges = np.roll(quad, -1, axis=0) - quad
if np.any(np.linalg.norm(edges, axis=1) < 2.0):
raise ValueError("Rectify corners are too close together")
crosses = np.cross(edges, np.roll(edges, -1, axis=0))
if np.any(np.abs(crosses) <= 1e-8) or not (
np.all(crosses > 0.0) or np.all(crosses < 0.0)
):
raise ValueError("Rectify corners must define one convex quadrilateral")
area = 0.5 * abs(
float(
np.dot(quad[:, 0], np.roll(quad[:, 1], -1))
- np.dot(quad[:, 1], np.roll(quad[:, 0], -1))
)
)
if area < min_area:
raise ValueError("Draw a larger Rectify quad")
return quad
def canonical_perspective_quad(points, min_area=1e-6):
"""Order four unordered points as source-oriented top-left through bottom-left."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
if points.shape != (4, 2) or not np.isfinite(points).all():
raise ValueError("Rectify requires four valid image corners")
distances = np.linalg.norm(points[:, None] - points[None, :], axis=2)
distances += np.eye(4) * 1e9
if float(distances.min()) < 2.0:
raise ValueError("Rectify corners are too close together")
center = points.mean(axis=0)
ring = points[np.argsort(np.arctan2(points[:, 1] - center[1], points[:, 0] - center[0]))]
ring = validate_perspective_quad(ring, min_area)
candidates = []
for winding in (ring, ring[::-1]):
for start in range(4):
candidate = np.roll(winding, -start, axis=0)
top = candidate[1] - candidate[0]
left = candidate[3] - candidate[0]
score = top[0] / np.linalg.norm(top) + left[1] / np.linalg.norm(left)
candidates.append((float(score), -candidate[0, 1], -candidate[0, 0], candidate))
return max(candidates, key=lambda item: item[:3])[3]
def preview_polygon(points):
"""Return a non-crossing preview hull for the current unordered clicks."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
if len(points) < 3:
return points
center = points.mean(axis=0)
return points[np.argsort(np.arctan2(points[:, 1] - center[1], points[:, 0] - center[0]))]
def interactive_preview_polygon(points, cursor):
"""Return the clicked points plus one distinct moving cursor point."""
vertices = list(points)
if (
cursor is not None
and vertices
and len(vertices) < 4
and math.dist(vertices[-1], cursor) >= 2.0
):
vertices.append(cursor)
return preview_polygon(vertices)
def perspective_quad_overlaps_image(points, image_size):
import numpy as np
width, height = image_size
if min(width, height) <= 0:
raise ValueError("The source image has no readable dimensions")
clipped = np.asarray(clip_polygon_halfplanes(
points, ((1, 0, 0), (-1, 0, width), (0, 1, 0), (0, -1, height))
))
if clipped.ndim != 2 or len(clipped) < 3:
return False
area = 0.5 * abs(
float(
np.dot(clipped[:, 0], np.roll(clipped[:, 1], -1))
- np.dot(clipped[:, 1], np.roll(clipped[:, 0], -1))
)
)
return area > 1e-6
def aspect_ratio_from_mouse(initial_ratio, horizontal_delta):
ratio = initial_ratio * math.pow(
2.0, float(horizontal_delta) / ASPECT_PIXELS_PER_DOUBLE
)
return min(max(ratio, MIN_INTERACTIVE_ASPECT), MAX_INTERACTIVE_ASPECT)
def snapped_aspect_ratio(aspect_ratio):
label, ratio = min(
ASPECT_PRESETS,
key=lambda preset: abs(math.log(aspect_ratio / preset[1])),
)
return ratio, label
def aspect_ratio_label(aspect_ratio, preset_label=None):
if preset_label is not None:
return preset_label
if aspect_ratio >= 1.0:
return f"{aspect_ratio:.2f} : 1"
return f"1 : {1.0 / aspect_ratio:.2f}"
def preview_draw_bounds(region_size, aspect_ratio):
region_width, region_height = (float(value) for value in region_size)
available_width = min(region_width * 0.65, 640.0)
available_height = min(region_height * 0.65, 640.0)
if aspect_ratio >= available_width / available_height:
width = available_width
height = width / aspect_ratio
else:
height = available_height
width = height * aspect_ratio
return (
(region_width - width) * 0.5,
(region_height - height) * 0.5,
width,
height,
)
def draw_preview_frame(bounds):
import gpu
from gpu_extras.batch import batch_for_shader
left, bottom, width, height = bounds
vertices = (
(left, bottom),
(left + width, bottom),
(left + width, bottom + height),
(left, bottom + height),
(left, bottom),
)
shader = gpu.shader.from_builtin("UNIFORM_COLOR")
batch = batch_for_shader(shader, "LINE_STRIP", {"pos": vertices})
gpu.state.blend_set("ALPHA")
gpu.state.line_width_set(2.0)
shader.bind()
shader.uniform_float("color", (0.12, 0.52, 1.0, 1.0))
batch.draw(shader)
gpu.state.line_width_set(1.0)
gpu.state.blend_set("NONE")
def draw_centered_text(text, center_x, baseline_y, size, color):
import blf
font_id = 0
blf.size(font_id, size)
width, _height = blf.dimensions(font_id, text)
x = center_x - width * 0.5
blf.position(font_id, x + 1.0, baseline_y - 1.0, 0.0)
blf.color(font_id, 0.0, 0.0, 0.0, 0.9)
blf.draw(font_id, text)
blf.position(font_id, x, baseline_y, 0.0)
blf.color(font_id, *color)
blf.draw(font_id, text)
def draw_aspect_hud(bounds, aspect_ratio, preset_label=None):
left, bottom, width, height = bounds
center_x = left + width * 0.5
draw_centered_text(
aspect_ratio_label(aspect_ratio, preset_label),
center_x,
bottom + height * 0.5 - 10.0,
22,
(0.25, 0.65, 1.0, 1.0),
)
draw_centered_text(
"Move horizontally • Ctrl snap • LMB confirm • Backspace edit",
center_x,
bottom - 26.0,
13,
(0.85, 0.9, 1.0, 1.0),
)
def create_perspective_preview_image(source_image, pixels):
image = bpy.data.images.new(
f"{image_base_name(source_image)}.rectify-preview",
width=PREVIEW_TEXTURE_SIZE,
height=PREVIEW_TEXTURE_SIZE,
alpha=True,
)
try:
image.colorspace_settings.name = "sRGB"
image.alpha_mode = "STRAIGHT"
image.pixels.foreach_set(pixels)
image.update()
except Exception:
bpy.data.images.remove(image, do_unlink=True)
raise
return image
def preview_texture_draw_options(blender_version):
if tuple(blender_version) >= (5, 0, 0):
return {"is_scene_linear_with_rec709_srgb_target": True}
return {}
def perspective_quad_aspect(quad):
"""Estimate an initial output aspect from the visible quad edge lengths."""
import numpy as np
quad = validate_perspective_quad(quad)
width = 0.5 * (
np.linalg.norm(quad[1] - quad[0]) + np.linalg.norm(quad[2] - quad[3])
)
height = 0.5 * (
np.linalg.norm(quad[2] - quad[1]) + np.linalg.norm(quad[3] - quad[0])
)
return min(
max(float(width / max(height, 1e-8)), MIN_INTERACTIVE_ASPECT),
MAX_INTERACTIVE_ASPECT,
)
def perspective_placement_bounds(quad, output_size, trim_bounds):
"""Map a rectified Alpha trim back to an approximate source-image rectangle."""
quad = validate_perspective_quad(quad)
source_left = math.floor(float(quad[:, 0].min()))
source_top = math.floor(float(quad[:, 1].min()))
source_right = math.ceil(float(quad[:, 0].max()))
source_bottom = math.ceil(float(quad[:, 1].max()))
output_width, output_height = (float(value) for value in output_size)
trim_left, trim_top, trim_right, trim_bottom = trim_bounds
source_width = source_right - source_left
source_height = source_bottom - source_top
return (
source_left + trim_left * source_width / output_width,
source_top + trim_top * source_height / output_height,
source_left + trim_right * source_width / output_width,
source_top + trim_bottom * source_height / output_height,
)
def extract_perspective_pixels(
source_pixels,
source_size,
quad,
aspect_ratio,
):
output_size = projective_output_size(quad, aspect_ratio)
pixels = warp_projective_pixels(
source_pixels, source_size, quad, output_size
)
pixels, trim_bounds = trim_pixels_to_alpha(pixels, (0, 0, *output_size))
trimmed_size = (
trim_bounds[2] - trim_bounds[0],
trim_bounds[3] - trim_bounds[1],
)
placement_bounds = perspective_placement_bounds(
quad, output_size, trim_bounds
)
return pixels, trimmed_size, placement_bounds
MIN_QUAD_AREA = 16.0
RECTIFY_TOOL_ID = "anyimage.rectify"
class RectifyImagePerspective(bpy.types.Operator):
bl_idname = "anyimage.rectify_image_perspective"
bl_label = "Rectify Image Perspective"
bl_description = "Rectify four image corners into a rectangular image"
bl_description = "Select four corners of an image area to correct its perspective into a rectangle."
bl_options = {"REGISTER", "UNDO"}
source_object_name: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
@@ -456,7 +192,7 @@ class RectifyImagePerspective(bpy.types.Operator):
return {"FINISHED"} if source_object is not None else {"CANCELLED"}
if source_object is None:
return {"CANCELLED"}
if video_total_frames(source_object.data) > 1:
if is_animated_image(source_object.data):
self.report(
{"WARNING"},
"Rectify only supports a still image",
@@ -621,7 +357,7 @@ class RectifyImagePerspective(bpy.types.Operator):
self.report({"ERROR"}, str(error))
return {"CANCELLED"}
try:
if video_total_frames(source_object.data) > 1:
if is_animated_image(source_object.data):
raise RuntimeError("Rectify only supports a still image")
quad = validate_perspective_quad(json.loads(self.quad_json))
if not perspective_quad_overlaps_image(
@@ -632,8 +368,6 @@ class RectifyImagePerspective(bpy.types.Operator):
"Rectify quad does not overlap the source image",
)
return {"CANCELLED"}
import numpy as np
source_pixels = self._source_pixels
if source_pixels is None or len(source_pixels) != len(
source_object.data.pixels
@@ -672,7 +406,7 @@ class RectifyTool(bpy.types.WorkSpaceTool):
bl_context_mode = "OBJECT"
bl_idname = RECTIFY_TOOL_ID
bl_label = "Rectify"
bl_description = "Rectify four image corners into a rectangular image"
bl_description = "Select four corners of an image area to correct its perspective into a rectangle."
bl_icon = "ops.sculpt.line_project"
bl_operator = RectifyImagePerspective.bl_idname
bl_keymap = image_edit_point_keymap(RectifyImagePerspective.bl_idname)

168
src/anyimage/operators/rectify_tool/preview.py Normal file
View File

@@ -0,0 +1,168 @@
"""Rectify preview images, overlays, and interactive aspect controls."""
import math
import bpy
from ...common.image import image_base_name
from .geometry import (
MAX_INTERACTIVE_ASPECT,
MIN_INTERACTIVE_ASPECT,
)
PREVIEW_TEXTURE_SIZE = 512
ASPECT_PIXELS_PER_DOUBLE = 240.0
ASPECT_PRESETS = (
("9 : 16", 9.0 / 16.0),
("3 : 4", 3.0 / 4.0),
("1 : 1", 1.0),
("4 : 3", 4.0 / 3.0),
("16 : 9", 16.0 / 9.0),
)
def preview_polygon(points):
"""Return a non-crossing preview hull for the current unordered clicks."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
if len(points) < 3:
return points
center = points.mean(axis=0)
return points[np.argsort(np.arctan2(points[:, 1] - center[1], points[:, 0] - center[0]))]
def interactive_preview_polygon(points, cursor):
"""Return the clicked points plus one distinct moving cursor point."""
vertices = list(points)
if (
cursor is not None
and vertices
and len(vertices) < 4
and math.dist(vertices[-1], cursor) >= 2.0
):
vertices.append(cursor)
return preview_polygon(vertices)
def aspect_ratio_from_mouse(initial_ratio, horizontal_delta):
ratio = initial_ratio * math.pow(
2.0, float(horizontal_delta) / ASPECT_PIXELS_PER_DOUBLE
)
return min(max(ratio, MIN_INTERACTIVE_ASPECT), MAX_INTERACTIVE_ASPECT)
def snapped_aspect_ratio(aspect_ratio):
label, ratio = min(
ASPECT_PRESETS,
key=lambda preset: abs(math.log(aspect_ratio / preset[1])),
)
return ratio, label
def aspect_ratio_label(aspect_ratio, preset_label=None):
if preset_label is not None:
return preset_label
if aspect_ratio >= 1.0:
return f"{aspect_ratio:.2f} : 1"
return f"1 : {1.0 / aspect_ratio:.2f}"
def preview_draw_bounds(region_size, aspect_ratio):
region_width, region_height = (float(value) for value in region_size)
available_width = min(region_width * 0.65, 640.0)
available_height = min(region_height * 0.65, 640.0)
if aspect_ratio >= available_width / available_height:
width = available_width
height = width / aspect_ratio
else:
height = available_height
width = height * aspect_ratio
return (
(region_width - width) * 0.5,
(region_height - height) * 0.5,
width,
height,
)
def draw_preview_frame(bounds):
import gpu
from gpu_extras.batch import batch_for_shader
left, bottom, width, height = bounds
vertices = (
(left, bottom),
(left + width, bottom),
(left + width, bottom + height),
(left, bottom + height),
(left, bottom),
)
shader = gpu.shader.from_builtin("UNIFORM_COLOR")
batch = batch_for_shader(shader, "LINE_STRIP", {"pos": vertices})
gpu.state.blend_set("ALPHA")
gpu.state.line_width_set(2.0)
shader.bind()
shader.uniform_float("color", (0.12, 0.52, 1.0, 1.0))
batch.draw(shader)
gpu.state.line_width_set(1.0)
gpu.state.blend_set("NONE")
def draw_centered_text(text, center_x, baseline_y, size, color):
import blf
font_id = 0
blf.size(font_id, size)
width, _height = blf.dimensions(font_id, text)
x = center_x - width * 0.5
blf.position(font_id, x + 1.0, baseline_y - 1.0, 0.0)
blf.color(font_id, 0.0, 0.0, 0.0, 0.9)
blf.draw(font_id, text)
blf.position(font_id, x, baseline_y, 0.0)
blf.color(font_id, *color)
blf.draw(font_id, text)
def draw_aspect_hud(bounds, aspect_ratio, preset_label=None):
left, bottom, width, height = bounds
center_x = left + width * 0.5
draw_centered_text(
aspect_ratio_label(aspect_ratio, preset_label),
center_x,
bottom + height * 0.5 - 10.0,
22,
(0.25, 0.65, 1.0, 1.0),
)
draw_centered_text(
"Move horizontally • Ctrl snap • LMB confirm • Backspace edit",
center_x,
bottom - 26.0,
13,
(0.85, 0.9, 1.0, 1.0),
)
def create_perspective_preview_image(source_image, pixels):
image = bpy.data.images.new(
f"{image_base_name(source_image)}.rectify-preview",
width=PREVIEW_TEXTURE_SIZE,
height=PREVIEW_TEXTURE_SIZE,
alpha=True,
)
try:
image.colorspace_settings.name = "sRGB"
image.alpha_mode = "STRAIGHT"
image.pixels.foreach_set(pixels)
image.update()
except Exception:
bpy.data.images.remove(image, do_unlink=True)
raise
return image
def preview_texture_draw_options(blender_version):
if tuple(blender_version) >= (5, 0, 0):
return {"is_scene_linear_with_rec709_srgb_target": True}
return {}

7
src/anyimage/operators/remove_background.py
View File

@@ -86,23 +86,20 @@ class RunBackgroundRemoval(JobOperatorBase, bpy.types.Operator):
require_environment()
model = configured_background_model(context)
require_model(model)
settings = context.scene.anyimage_settings
parameters = {
"input": str(require_input_path(self.input_path)),
"model": model,
"device": production_device().lower(),
"video_frames": settings.video_frames,
"start_frame": self._image_target.start_frame,
}
if getattr(self, "_hdr_input", None) is not None:
parameters.update(output_kind="alpha", video_frames=1, start_frame=1)
parameters["output_kind"] = "alpha"
return parameters
def response(self, _context, result):
if getattr(self, "_hdr_input", None) is not None:
self._hdr_input.apply(self._image_target, result.file("alpha"), prepare_undo=True)
else:
self._image_target.apply(result.directory)
self._image_target.apply(result.file("foreground"))
if "FINISHED" not in bpy.ops.ed.undo_push(message="Remove Background"):
raise RuntimeError("Unable to create the Remove Background Undo step")
return "Image background is removed"

5
src/anyimage/operators/upscale.py
View File

@@ -115,18 +115,15 @@ class UpscaleImage(JobOperatorBase, bpy.types.Operator):
def request(self, context):
require_environment()
require_model(self.model)
settings = context.scene.anyimage_settings
return {
"input": str(require_input_path(self.input_path)),
"model": self.model,
"device": production_device().lower(),
"video_frames": settings.video_frames,
"start_frame": self._image_target.start_frame,
"max_input_size": int(self.max_input_size),
}
def response(self, _context, result):
self._image_target.apply(result.directory, frame_prefix="upscale.")
self._image_target.apply(result.file("upscale"))
model_label = upscale_model_label(self.model)
return f"Image is upscaled with {model_label}"