icrdr/AnyImage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add new functionality for image rectification and depth metadata handling

Browse Source 
- Implemented a preview tool for rectifying images with interactive aspect controls in `preview.py`.
- Added tests for depth metadata loading to ensure consistent schema usage in `test_depth_metadata.py`.
- Developed comprehensive tests for cutout calibration processes in `test_cutout_calibration.py`, including stability checks and depth calibration.
- Created tests for frame compositing operations in `test_frame_compositing.py`, ensuring correct handling of depth and transparency.
- Introduced material slot inheritance functionality in `material_slots.py` to manage geometry material assignments.
- Built shared shader nodes for material processing in `shader.py`, including roughness and bump mapping setups.
This commit is contained in:
马南
2026-09-12 19:20:06 +08:00
parent e8268504c2
commit 938d95db0f
175 changed files with 3812 additions and 6864 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
src/anyimage/assets/O_AnyImage.blend LFS
View File

Binary file not shown.

2
src/anyimage/blender_manifest.toml
View File

@@ -1,7 +1,7 @@
schema_version = "1.0.0"
id = "anyimage"
version = "0.13.2"
version = "0.13.3"
name = "AnyImage"
tagline = "Build spatial structures from images, fast"
maintainer = "OmooLab"

4
src/anyimage/common/ai.py
View File

@@ -69,12 +69,10 @@ def require_model(model_key):
return model
def moge2_parameters(settings, path, model, resolution_level, start_frame):
def moge2_parameters(path, model, resolution_level):
return {
"input": str(path),
"model": model,
"resolution_level": int(resolution_level),
"video_frames": settings.video_frames,
"start_frame": start_frame,
"device": production_device().lower(),
}

4
src/anyimage/common/color_image.py
View File

@@ -7,12 +7,12 @@ from pathlib import Path
import bpy
from .image import cleanup_image_input, image_base_name, image_is_packed, image_rgba, is_animated_image, video_total_frames
from .image import cleanup_image_input, image_base_name, image_is_packed, image_rgba, is_animated_image
def require_static_color_image(image):
"""Reject animated inputs before preparing material resources."""
if is_animated_image(image) or video_total_frames(image) != 1:
if is_animated_image(image):
raise ValueError("Material conversion supports static images only")

284
src/anyimage/common/image.py
View File

@@ -1,14 +1,12 @@
"""Shared Blender Image and Image Empty lifecycle helpers."""
import math
import re
import shutil
import tempfile
from pathlib import Path
import bpy
from ..server.media.input import validate_input_path
VIEW_TRANSFORM_COLOR_SPACES = (
("ACES 1.3", "ACES 1.3 sRGB"),
@@ -36,19 +34,6 @@ MAX_PROJECTIVE_OUTPUT_PIXELS = 16_777_216
PROJECTIVE_CHUNK_ROWS = 256
VIDEO_NAME_SUFFIXES = (
".mp4",
".mov",
".avi",
".mkv",
".webm",
".m4v",
".mts",
".mpg",
".mpeg",
)
def is_image_empty(candidate):
return bool(
candidate
@@ -290,21 +275,10 @@ def warp_projective_pixels(source_pixels, source_size, quad, output_size):
return np.flipud(straight_rgba(output)).ravel()
def is_animated_image(image):
return getattr(image, "source", None) in {"MOVIE", "SEQUENCE"}
def video_total_frames(image):
"""Return the total animated frame count, or 1 for a still image."""
source = getattr(image, "source", None)
if source == "MOVIE":
return int(getattr(image, "frame_duration", 1) or 1)
if source == "SEQUENCE":
return len(collect_sequence_frames(image)) or 1
return 1
def image_user_settings(source_object):
"""Return an image owner's playback settings as a four-item tuple."""
image_user = getattr(source_object, "image_user", None)
@@ -318,28 +292,11 @@ def image_user_settings(source_object):
)
def extract_frame_range(image, image_user):
"""Return the clamped (start_frame, frame_count) range extracted from the source."""
total_frames = video_total_frames(image)
if total_frames <= 1:
return 1, total_frames
frame_offset = int(getattr(image_user, "frame_offset", 0) or 0)
image_frames = int(getattr(image_user, "frame_duration", 0) or 0)
frame_count = (
min(image_frames, total_frames)
if image_frames >= 1
else total_frames
)
start_frame = min(1 + frame_offset, total_frames - frame_count + 1)
start_frame = max(start_frame, 1)
return start_frame, frame_count
def image_path(image):
filepath_from_user = getattr(image, "filepath_from_user", None)
filepath = (
filepath_from_user()
if callable(filepath_from_user) and getattr(image, "source", None) != "SEQUENCE"
if callable(filepath_from_user)
else getattr(image, "filepath", "")
)
if not filepath:
@@ -371,7 +328,7 @@ def cleanup_image_input(path, temporary):
def image_base_name(image):
name = image.name
lower_name = name.lower()
for suffix in IMAGE_NAME_SUFFIXES + VIDEO_NAME_SUFFIXES:
for suffix in IMAGE_NAME_SUFFIXES:
suffix_index = lower_name.rfind(suffix)
if suffix_index <= 0:
continue
@@ -409,137 +366,35 @@ def save_packed_image(image, _scene):
return output_path
def collect_sequence_frames(image):
"""Return the numbered frame files of a SEQUENCE image, or [] when absent."""
source_path = image_path(image)
if not is_animated_image(image) or not source_path.is_file():
return []
match = re.match(
r"^(?P<prefix>.*?)(?P<digits>\d+)(?P<suffix>\.[^.]+)$",
source_path.name,
)
if match is None:
return []
prefix = match.group("prefix")
suffix = match.group("suffix").lower()
frames = []
for path in source_path.parent.iterdir():
frame_match = re.match(
r"^(?P<prefix>.*?)(?P<digits>\d+)(?P<suffix>\.[^.]+)$",
path.name,
)
if (
frame_match is not None
and frame_match.group("prefix") == prefix
and frame_match.group("suffix").lower() == suffix
):
frames.append((int(frame_match.group("digits")), path))
return [path for _digits, path in sorted(frames)]
def prepare_image_input(image, scene):
source_path = image_path(image)
"""Prepare one still image for a Server job."""
if is_animated_image(image):
raise RuntimeError("Only still images are supported")
if image_is_packed(image):
return save_packed_image(image, scene), True
if getattr(image, "source", None) == "MOVIE":
if not source_path.is_file():
raise RuntimeError(
f"Source video file does not exist: {source_path}"
)
return source_path, False
if getattr(image, "source", None) == "SEQUENCE":
validate_input_path(source_path, images_only=True)
sequence_frames = collect_sequence_frames(image)
if sequence_frames:
temporary_directory = Path(tempfile.mkdtemp(prefix="anyimage-input-"))
try:
for index, frame_path in enumerate(sequence_frames):
suffix = frame_path.suffix.lower() or ".jpg"
shutil.copy2(
frame_path,
temporary_directory / f"frame.{index + 1:04d}{suffix}",
)
except Exception:
shutil.rmtree(temporary_directory, ignore_errors=True)
raise
return temporary_directory, True
source_path = image_path(image)
if not source_path.is_file():
raise RuntimeError(
f"Source image file does not exist: {source_path}"
)
raise RuntimeError(f"Source image file does not exist: {source_path}")
return source_path, False
def configure_image_user(
image_user,
frame_count=None,
frame_start=1,
frame_offset=0,
use_cyclic=None,
):
"""Map the image to scene frames and enable auto refresh."""
if image_user is None:
return
if hasattr(image_user, "frame_start"):
image_user.frame_start = frame_start
if hasattr(image_user, "frame_offset"):
image_user.frame_offset = frame_offset
if hasattr(image_user, "use_auto_refresh"):
image_user.use_auto_refresh = True
if use_cyclic is not None and hasattr(image_user, "use_cyclic"):
image_user.use_cyclic = use_cyclic
if frame_count is not None and hasattr(image_user, "frame_duration"):
image_user.frame_duration = frame_count
def pack_sequence_if_possible(image, expected_frames):
"""Pack a frame sequence when Blender supports it; otherwise keep external files."""
if expected_frames <= 1 or not image.has_data:
return False
def load_image_edit_result(source_image, output_path):
"""Load and pack the image file returned by a Job."""
output_path = Path(output_path)
if not output_path.is_file():
raise RuntimeError(f"Color result image does not exist: {output_path}")
result_image = None
try:
image.pack()
except (OSError, RuntimeError):
return False
if len(getattr(image, "packed_files", ())) >= expected_frames:
return True
try:
image.unpack(method="REMOVE")
except (OSError, RuntimeError):
pass
return False
def sorted_frame_paths(directory, prefix=None):
"""Return image files in a directory sorted by their trailing frame number."""
def frame_key(path):
match = re.search(r"(\d+)(?=\.[^.]+$)", path.name)
return (int(match.group(1)) if match is not None else 0, path.name)
return sorted(
(
path
for path in Path(directory).iterdir()
if path.is_file()
and path.suffix.lower() in IMAGE_NAME_SUFFIXES
and (prefix is None or path.stem.startswith(prefix))
),
key=frame_key,
)
def force_sequence_source(image):
"""Ensure a loaded image is a SEQUENCE datablock for its numbered frames."""
if getattr(image, "source", None) == "SEQUENCE":
return image
try:
image.source = "SEQUENCE"
except (OSError, RuntimeError, TypeError) as error:
raise RuntimeError(
"Blender cannot load the frame sequence"
) from error
if getattr(image, "source", None) != "SEQUENCE":
raise RuntimeError("Blender cannot load the frame sequence")
return image
result_image = bpy.data.images.load(str(output_path), check_existing=False)
result_image.colorspace_settings.name = source_image.colorspace_settings.name
result_image.alpha_mode = source_image.alpha_mode
result_image.pack()
result_image.name = source_image.name
except (OSError, RuntimeError, TypeError, ValueError) as error:
if result_image is not None:
bpy.data.images.remove(result_image, do_unlink=True)
raise RuntimeError("Unable to load the color result") from error
return result_image
def load_normal_result_image(path, source_object):
@@ -556,22 +411,6 @@ def color_alpha_mode(color_space):
return "STRAIGHT" if color_space in dict(VIEW_TRANSFORM_COLOR_SPACES).values() else "PREMUL"
def configure_color_alpha(image):
"""Apply the presentation mode to an owned Color image."""
alpha_mode = color_alpha_mode(image.colorspace_settings.name)
if image.alpha_mode != alpha_mode:
pixels = (
image_pixels(image)
if getattr(image, "source", None) == "GENERATED"
or getattr(image, "is_dirty", False)
else None
)
image.alpha_mode = alpha_mode
if pixels is not None:
image.pixels.foreach_set(pixels)
image.update()
def create_image_edit_result(source_image, pixels, output_size):
"""Create a packed Image datablock from edited RGBA pixels."""
output_width, output_height = (int(value) for value in output_size)
@@ -680,11 +519,6 @@ def replace_empty_image(source_object, result_image, *, placement_bounds=None,
image_user = getattr(source_object, "image_user", None)
original_timing = image_user_settings(source_object) if image_user is not None else None
original_auto_refresh = getattr(image_user, "use_auto_refresh", False)
frame_count = (
video_total_frames(result_image)
if is_animated_image(result_image)
else None
)
alignment = (
empty_display_alignment(
source_object,
@@ -695,13 +529,6 @@ def replace_empty_image(source_object, result_image, *, placement_bounds=None,
if placement_bounds is not None
else None
)
if frame_count is not None:
(
original_frame_start,
_original_frame_offset,
_original_frame_duration,
original_use_cyclic,
) = image_user_settings(source_object)
source_image.use_fake_user = True
original_alignment = (
@@ -720,13 +547,6 @@ def replace_empty_image(source_object, result_image, *, placement_bounds=None,
final_image = source_image
source_object.data = final_image
source_object.use_empty_image_alpha = True
if frame_count is not None:
configure_image_user(
image_user,
frame_count,
frame_start=original_frame_start,
use_cyclic=original_use_cyclic,
)
if alignment is not None:
source_object.empty_display_size = alignment[0]
source_object.empty_image_offset = alignment[1]
@@ -751,64 +571,6 @@ def replace_empty_image(source_object, result_image, *, placement_bounds=None,
bpy.data.images.remove(result_image)
def _load_packed_result_image(
source_image,
output_directory,
*,
frame_prefix,
result_name,
alpha_mode,
):
output_directory = Path(output_directory)
color_paths = sorted_frame_paths(output_directory, frame_prefix)
if not color_paths:
raise RuntimeError(
f"Color result has no images: {output_directory}"
)
source_color_space = source_image.colorspace_settings.name
animated = is_animated_image(source_image)
try:
result_image = bpy.data.images.load(
str(color_paths[0]),
check_existing=False,
)
result_image.colorspace_settings.name = source_color_space
result_image.alpha_mode = alpha_mode
if animated:
force_sequence_source(result_image)
except (OSError, RuntimeError, TypeError, ValueError) as error:
if "result_image" in locals():
bpy.data.images.remove(result_image, do_unlink=True)
raise RuntimeError("Unable to load the color result") from error
try:
if animated:
pack_sequence_if_possible(result_image, len(color_paths))
else:
result_image.pack()
result_image.name = result_name
except (OSError, RuntimeError, TypeError, ValueError) as error:
bpy.data.images.remove(result_image, do_unlink=True)
raise RuntimeError("Unable to prepare the color result") from error
return result_image
def load_image_edit_result(
source_image,
output_directory,
*,
frame_prefix="image.",
):
"""Load one packed Job result that directly edits the source Image."""
return _load_packed_result_image(
source_image,
output_directory,
frame_prefix=frame_prefix,
result_name=source_image.name,
alpha_mode=source_image.alpha_mode,
)
def image_empty_bounds(source_object):
image_width, image_height = source_object.data.size
if image_width <= 0 or image_height <= 0:

24
src/anyimage/common/image_target.py
View File

@@ -6,8 +6,6 @@ from uuid import uuid4
import bpy
from .image import (
configure_image_user,
extract_frame_range,
image_content_state,
image_user_settings,
is_animated_image,
@@ -16,7 +14,6 @@ from .image import (
prepare_image_input,
replace_empty_image,
restore_image_content,
video_total_frames,
)
@@ -84,7 +81,6 @@ def replace_texture_image(node, result_image):
original_timing = image_user_settings(node)
original_auto_refresh = node.image_user.use_auto_refresh
original_content = None
frame_count = video_total_frames(result_image) if is_animated_image(result_image) else None
try:
if has_other_image_user(node):
final_image = result_image
@@ -94,13 +90,6 @@ def replace_texture_image(node, result_image):
restore_image_content(source_image, result_content)
final_image = source_image
node.image = final_image
if frame_count is not None:
configure_image_user(
node.image_user,
frame_count,
frame_start=original_timing[0],
use_cyclic=original_timing[3],
)
except Exception:
if original_content is not None:
restore_image_content(source_image, original_content)
@@ -122,8 +111,6 @@ class ImageEditTarget:
owner: object
image: object
tree: object = None
start_frame: int = 1
frame_count: int = 1
node_identity: str = ""
@classmethod
@@ -132,7 +119,8 @@ class ImageEditTarget:
if owner is None:
raise RuntimeError("Select an Image Empty or a material Image Texture")
image = owner_image(owner)
start_frame, frame_count = extract_frame_range(image, owner.image_user)
if is_animated_image(image):
raise RuntimeError("Only still images are supported")
tree = None if is_image_empty(owner) else owner.id_data
identity = ""
if tree is not None:
@@ -141,7 +129,7 @@ class ImageEditTarget:
identity = uuid4().hex
owner["anyimage_identity"] = identity
return cls(
owner, image, tree, start_frame, frame_count, identity,
owner, image, tree, identity,
)
def validate(self):
@@ -165,13 +153,11 @@ class ImageEditTarget:
def prepare(self, context):
self.validate()
if video_total_frames(self.image) > 1:
context.scene.anyimage_settings.video_frames = self.frame_count
return prepare_image_input(self.image, context.scene)
def apply(self, directory, *, frame_prefix="image."):
def apply(self, output_path):
self.validate()
result_image = load_image_edit_result(self.image, directory, frame_prefix=frame_prefix)
result_image = load_image_edit_result(self.image, output_path)
return self.commit(result_image)
def commit(self, result_image, *, isolate_shared=False):

4
src/anyimage/common/viewport.py
View File

@@ -10,7 +10,7 @@ from .image import (
image_empty_bounds,
is_image_empty,
require_image_empty,
video_total_frames,
is_animated_image,
)
from .selection import ImageEditWarning, SelectionPath
@@ -1019,7 +1019,7 @@ class ImageGesture:
return {"CANCELLED"}
if source_object is None:
return {"CANCELLED"}
if video_total_frames(source_object.data) > 1:
if is_animated_image(source_object.data):
self.report({"WARNING"}, "Image gestures only support a still image")
return {"CANCELLED"}
self.source_object_name = source_object.name

28
src/anyimage/operators/__init__.py
View File

@@ -1,17 +1,9 @@
from .remove_background import RemoveImageBackground, RunBackgroundRemoval
from .cutout_tool import (
CutoutSelectionToShape,
SelectCutoutSelection,
)
from .frame_tool import FrameImages
from .cutout_tool import CLASSES as CUTOUT_CLASSES
from .frame_tool import CLASSES as FRAME_CLASSES
from .mask_tool import EditImageAlpha
from .rectify_tool import RectifyImagePerspective
from .convert_to_plane import (
ConvertToDepthPlane,
ConvertToReliefPlane,
ConvertToPlane,
GenerateDepthPlane,
)
from .rectify_tool import CLASSES as RECTIFY_CLASSES
from .convert_to_plane import CLASSES as PLANE_CLASSES
from .ai_setup import (
ClearModels,
DownloadModel,
@@ -25,21 +17,17 @@ from .convert_to_panorama import CLASSES as PANORAMA_CLASSES
CLASSES = (
*PANORAMA_CLASSES,
*PLANE_CLASSES,
*CUTOUT_CLASSES,
SetupAIEnvironment,
OpenAIEnvironmentSettings,
DownloadModel,
DownloadRequiredModels,
GenerateDepthPlane,
CutoutSelectionToShape,
SelectCutoutSelection,
RunBackgroundRemoval,
ClearModels,
FrameImages,
*FRAME_CLASSES,
EditImageAlpha,
RectifyImagePerspective,
*RECTIFY_CLASSES,
RemoveImageBackground,
UpscaleImage,
ConvertToPlane,
ConvertToDepthPlane,
ConvertToReliefPlane,
)

47
src/anyimage/operators/ai_setup.py
View File

@@ -21,11 +21,8 @@ class SetupAIEnvironment(bpy.types.Operator):
bl_description = "Install the environment and required AI models"
use_mirror: bpy.props.BoolProperty(
name="Use Mirror",
description=(
"Use the China mirrors for uv, Python and Python packages; "
"model download sources are unchanged"
),
name="Use China Mirror",
description="Download Python and packages from China mirrors",
default=False,
)
@@ -34,36 +31,24 @@ class SetupAIEnvironment(bpy.types.Operator):
return not runtime.server_busy()
def invoke(self, context, _event):
return context.window_manager.invoke_props_dialog(self, width=440)
return context.window_manager.invoke_props_dialog(
self, width=440, title="Set Up AI", confirm_text="Install"
)
def draw(self, _context):
layout = self.layout
status = ai_status()
if not status["environment_ready"]:
required_size = sum(
model_download_size(model_key)
for model_key in DEFAULT_MODEL_KEYS
model_download_size(model_key) for model_key in DEFAULT_MODEL_KEYS
)
layout.label(
text="AnyImage AI features run in an isolated Python environment."
)
layout.label(
text="This installs Python, ONNX Runtime and supporting packages."
text="Set up AI features for depth, background removal and upscaling."
)
layout.label(
text=(
"It also downloads MoGe-2 ViT-S Normal, BiRefNet Lite and "
"Real-ESRGAN General WDN x4v3 "
f"({format_download_size(required_size)})."
)
)
layout.label(
text="Allow about 600 MB of downloads and 1 GB of disk space."
)
layout.label(
text=(
"Installation usually takes 5–20 minutes, "
"depending on network speed."
"Downloads the runtime and required models "
f"({format_download_size(required_size)} for models)."
)
)
layout.separator(type="LINE")
@@ -71,7 +56,7 @@ class SetupAIEnvironment(bpy.types.Operator):
return
if status["ready"]:
layout.label(text="The AI Environment is ready.", icon="CHECKMARK")
layout.label(text="AI Server is ready.", icon="CHECKMARK")
return
labels = []
@@ -83,12 +68,13 @@ class SetupAIEnvironment(bpy.types.Operator):
for model_key in status["missing_models"]
)
layout.label(
text="The AI Environment is already installed.",
text="AI environment installed.",
icon="CHECKMARK",
)
layout.label(text=f"Required models to download: {', '.join(labels)}")
layout.label(text=f"Download size: {format_download_size(missing_size)}")
layout.label(text="The download usually takes 2–10 minutes.")
layout.label(text=f"Download: {', '.join(labels)}")
layout.label(
text=f"{format_download_size(missing_size)} · Usually 2–10 minutes."
)
def execute(self, _context):
status = ai_status()
@@ -159,7 +145,8 @@ class DownloadRequiredModels(JobOperatorBase, bpy.types.Operator):
class ClearModels(bpy.types.Operator):
bl_idname = "anyimage.clear_models"
bl_label = "Clear Models"
bl_label = "Unload Models"
bl_description = "Unload AI models to free memory. Downloaded model files are kept."
def execute(self, _context):
try:

3
src/anyimage/operators/clipboard_image/__init__.py
View File

@@ -1,11 +1,12 @@
from .operators import (
CLASSES,
PasteClipboardImage,
TrackNativeCopy,
register_keymaps,
unregister_keymaps,
)
CLASSES = (TrackNativeCopy, PasteClipboardImage)
__all__ = (
"CLASSES",
"PasteClipboardImage",

3
src/anyimage/operators/clipboard_image/operators.py
View File

@@ -148,9 +148,6 @@ class PasteClipboardImage(bpy.types.Operator):
return {"FINISHED"}
CLASSES = (TrackNativeCopy, PasteClipboardImage)
def unregister_keymaps():
global _native_copy_token
for keymap, keymap_item in _keymap_items:

12
src/anyimage/operators/convert_to_panorama/operators.py
View File

@@ -3,7 +3,7 @@
import bpy
from ...common.ai import invoke_ai_setup_if_needed, production_device, report_ai_error, require_environment, require_input_path, require_model
from ...common.image import is_image_empty, video_total_frames
from ...common.image import is_image_empty, is_animated_image
from ...common.color_image import prepare_material_color_input, material_analysis_input, cleanup_material_color_input
from ...preferences import addon_preferences, configured_max_ai_input_size
from ...preferences import configured_geometry_model
@@ -26,7 +26,7 @@ class GeneratePanorama(JobOperatorBase, bpy.types.Operator):
source_object_name: bpy.props.StringProperty(options=_HIDDEN)
source_identity: bpy.props.StringProperty(options=_HIDDEN)
image_identity: bpy.props.StringProperty(options=_HIDDEN)
mesh_detail: bpy.props.IntProperty(default=7, min=0, max=8, options=_HIDDEN)
mesh_detail: bpy.props.IntProperty(default=8, min=0, max=8, options=_HIDDEN)
model: bpy.props.StringProperty(default=DEFAULT_GEOMETRY_MODEL_KEY, options=_HIDDEN)
resolution_level: bpy.props.IntProperty(default=5, min=0, max=9, options=_HIDDEN)
max_input_size: bpy.props.IntProperty(default=DEFAULT_MAX_AI_INPUT_SIZE,
@@ -39,7 +39,7 @@ class GeneratePanorama(JobOperatorBase, bpy.types.Operator):
"input": str(require_input_path(self.input_path)),
"device": production_device().lower(), "model": self.model,
"resolution_level": int(self.resolution_level),
"max_input_size": int(self.max_input_size), "video_frames": 1, "start_frame": 1,
"max_input_size": int(self.max_input_size),
}
def response(self, context, result):
@@ -53,10 +53,10 @@ class GeneratePanorama(JobOperatorBase, bpy.types.Operator):
class ConvertToPanorama(bpy.types.Operator):
bl_idname = "anyimage.convert_to_panorama"
bl_label = "Convert to Panorama"
bl_description = "Convert a static 2:1 equirectangular Image Empty into a radial depth surface"
bl_description = "Turn a still 2:1 panoramic image into a surrounding 3D surface using AI-estimated depth."
bl_options = {"REGISTER", "UNDO"}
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=7, min=0, max=8)
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=8, min=0, max=8)
@classmethod
def poll(cls, context):
@@ -68,7 +68,7 @@ class ConvertToPanorama(bpy.types.Operator):
source = context.object
image = source.data
width, height = image.size
if image.source in {"MOVIE", "SEQUENCE"} or video_total_frames(image) != 1 or height <= 0 or width != height * 2:
if is_animated_image(image) or height <= 0 or width != height * 2:
self.report({"ERROR"}, "Panorama conversion requires a static 2:1 equirectangular image")
return {"CANCELLED"}
setup = invoke_ai_setup_if_needed(self)

3
src/anyimage/operators/convert_to_plane/__init__.py
View File

@@ -7,7 +7,10 @@ from .operators import (
GenerateDepthPlane,
)
CLASSES = (GenerateDepthPlane, ConvertToPlane, ConvertToDepthPlane, ConvertToReliefPlane)
__all__ = (
"CLASSES",
"ConvertToDepthPlane",
"ConvertToReliefPlane",
"ConvertToPlane",

11
src/anyimage/operators/convert_to_plane/operators.py
View File

@@ -75,18 +75,14 @@ class GenerateDepthPlane(JobOperatorBase, bpy.types.Operator):
parameters = {
"input": str(input_path),
"device": production_device().lower(),
"video_frames": 1,
"start_frame": 1,
"max_input_size": int(self.max_input_size),
"plane_type": self.plane_type,
}
parameters.update(
moge2_parameters(
context.scene.anyimage_settings,
input_path,
self.model,
self.resolution_level,
1,
)
)
return parameters
@@ -175,7 +171,10 @@ class ConvertToPlane(bpy.types.Operator):
class ConvertToDepthPlane(bpy.types.Operator):
bl_idname = "anyimage.convert_to_depth_plane"
bl_label = "Convert to Depth Plane"
bl_description = "Convert the complete Image Empty into a camera-projected depth plane"
bl_description = (
"Use AI-estimated depth to turn the image into a 3D surface "
"that follows the source image's perspective."
)
bl_options = {"REGISTER", "UNDO"}
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=6, min=0, max=10)
@@ -193,7 +192,7 @@ class ConvertToDepthPlane(bpy.types.Operator):
class ConvertToReliefPlane(bpy.types.Operator):
bl_idname = "anyimage.convert_to_relief_plane"
bl_label = "Convert to Relief Plane"
bl_description = "Convert the complete Image Empty into a fixed-base relief plane"
bl_description = "Use AI-estimated depth to create a raised relief on a flat base."
bl_options = {"REGISTER", "UNDO"}
mesh_detail: bpy.props.IntProperty(name="Subdivide", default=6, min=0, max=10)

3
src/anyimage/operators/cutout_tool/__init__.py
View File

@@ -1,6 +1,9 @@
from .operators import CutoutSelectionToShape, CutoutTool, SelectCutoutSelection
CLASSES = (CutoutSelectionToShape, SelectCutoutSelection)
__all__ = (
"CLASSES",
"CutoutSelectionToShape",
"CutoutTool",
"SelectCutoutSelection",

396
src/anyimage/operators/cutout_tool/balloon.py Normal file
View File

@@ -0,0 +1,396 @@
"""Compute Poisson and Teddy height profiles for Cutout meshes."""
import math
DEFAULT_INFLATION = 0.45
def distances_to_contour(points, contour):
import numpy as np
points = np.asarray(points, dtype=np.float64)
contour = np.asarray(contour, dtype=np.float64)
squared = np.full(len(points), np.inf, dtype=np.float64)
for start, end in zip(contour, np.roll(contour, -1, axis=0)):
segment = end - start
length_squared = float(segment.dot(segment))
if length_squared <= 1e-16:
continue
factor = np.clip(
(points - start) @ segment / length_squared,
0.0,
1.0,
)
nearest = start + factor[:, None] * segment
squared = np.minimum(
squared,
np.sum((points - nearest) ** 2, axis=1),
)
return np.sqrt(squared)
def chordal_axis(contour, epsilon=1e-9):
import numpy as np
from mathutils import Vector
from mathutils.geometry import delaunay_2d_cdt
contour = np.asarray(contour, dtype=np.float64)
boundary_count = len(contour)
boundary_edges = [
(index, (index + 1) % boundary_count) for index in range(boundary_count)
]
result = delaunay_2d_cdt(
[Vector(point) for point in contour],
boundary_edges,
[list(range(boundary_count))],
1,
epsilon,
)
points = np.asarray(
[tuple(point) for point in result[0]],
dtype=np.float64,
)
triangles = [tuple(face) for face in result[2] if len(face) == 3]
boundary = {tuple(sorted(edge)) for edge in boundary_edges}
node_indices = {}
nodes = []
segments = set()
def edge_node(edge):
key = ("edge", *tuple(sorted(edge)))
if key not in node_indices:
node_indices[key] = len(nodes)
nodes.append((points[edge[0]] + points[edge[1]]) * 0.5)
return node_indices[key]
for face_index, face in enumerate(triangles):
edges = [
(face[0], face[1]),
(face[1], face[2]),
(face[2], face[0]),
]
internal = [edge for edge in edges if tuple(sorted(edge)) not in boundary]
internal_nodes = [edge_node(edge) for edge in internal]
if len(internal_nodes) == 2:
segments.add(tuple(sorted(internal_nodes)))
elif len(internal_nodes) == 3:
key = ("face", face_index)
center_index = len(nodes)
node_indices[key] = center_index
nodes.append(points[list(face)].mean(axis=0))
for midpoint_index in internal_nodes:
segments.add(tuple(sorted((center_index, midpoint_index))))
if not nodes:
center = contour.mean(axis=0)
return (
np.asarray((center,), dtype=np.float64),
[],
np.asarray(
(distances_to_contour((center,), contour)[0],),
dtype=np.float64,
),
)
nodes = np.asarray(nodes, dtype=np.float64)
radii = distances_to_contour(nodes, contour)
active = set(segments)
while len(active) > 1:
degree = [0] * len(nodes)
for start, end in active:
degree[start] += 1
degree[end] += 1
removable = []
for edge in active:
start, end = edge
leaf = start if degree[start] == 1 else end if degree[end] == 1 else None
if leaf is None:
continue
length = float(np.linalg.norm(nodes[start] - nodes[end]))
if length < max(radii[leaf] * 0.5, epsilon):
removable.append(edge)
if not removable:
break
active.difference_update(removable)
if active:
used = sorted({index for edge in active for index in edge})
remap = {old: new for new, old in enumerate(used)}
nodes = nodes[used]
radii = radii[used]
active = {tuple(sorted((remap[start], remap[end]))) for start, end in active}
else:
widest = int(np.argmax(radii))
nodes = nodes[[widest]]
radii = radii[[widest]]
return nodes, sorted(active), radii
def smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
iterations=2,
factor=0.3,
):
import numpy as np
values = np.asarray(heights, dtype=np.float64).copy()
boundary_mask = np.asarray(boundary_mask, dtype=bool)
neighbors = [set() for _index in range(len(values))]
for face in planar_faces:
for start, end in zip(face, face[1:] + face[:1]):
neighbors[start].add(end)
neighbors[end].add(start)
for _iteration in range(iterations):
updated = values.copy()
for index in np.flatnonzero(~boundary_mask):
adjacent = [
neighbor for neighbor in neighbors[index] if not boundary_mask[neighbor]
]
if adjacent:
average = float(values[adjacent].mean())
updated[index] = values[index] * (1.0 - factor) + average * factor
updated[boundary_mask] = 0.0
values = updated
return values
def teddy_balloon_heights(
planar_points,
planar_faces,
contour,
boundary_mask,
inflation,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
boundary_mask = np.asarray(boundary_mask, dtype=bool)
nodes, segments, radii = chordal_axis(contour)
squared_heights = np.zeros(len(points), dtype=np.float64)
for start, end in segments:
axis_start = nodes[start]
axis_end = nodes[end]
axis = axis_end - axis_start
length_squared = float(axis.dot(axis))
if length_squared <= 1e-16:
continue
factors = np.clip(
(points - axis_start) @ axis / length_squared,
0.0,
1.0,
)
centers = axis_start + factors[:, None] * axis
local_radii = radii[start] + factors * (radii[end] - radii[start])
distance_squared = np.sum((points - centers) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
local_radii * local_radii - distance_squared,
)
degree = np.zeros(len(nodes), dtype=np.int64)
for start, end in segments:
degree[start] += 1
degree[end] += 1
for index in np.flatnonzero(degree != 2):
distance_squared = np.sum((points - nodes[index]) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
radii[index] * radii[index] - distance_squared,
)
missing = (~boundary_mask) & (squared_heights <= 0.0)
if np.any(missing):
fallback = distances_to_contour(points[missing], contour)
squared_heights[missing] = fallback * fallback
heights = inflation * np.sqrt(np.maximum(squared_heights, 0.0))
heights[boundary_mask] = 0.0
return smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
)
def cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
tolerance=1e-8,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
faces = np.asarray(planar_faces, dtype=np.int64)
if faces.size == 0:
faces = np.empty((0, 3), dtype=np.int64)
if faces.ndim != 2 or faces.shape[1] != 3:
raise ValueError("Balloon profile faces must be triangles")
boundary_mask = np.asarray(boundary_mask, dtype=bool)
triangles = points[faces]
a, b, c = (triangles[:, index] for index in range(3))
cross = np.abs(
(b[:, 0] - a[:, 0]) * (c[:, 1] - a[:, 1])
- (b[:, 1] - a[:, 1]) * (c[:, 0] - a[:, 0])
)
usable = cross > 1e-14
faces = faces[usable]
a, b, c = a[usable], b[usable], c[usable]
cross = cross[usable]
loads = np.bincount(
faces.ravel(),
weights=np.repeat(cross / 6.0, 3),
minlength=len(points),
)
edges = np.concatenate(
(
faces[:, (1, 2)],
faces[:, (0, 2)],
faces[:, (0, 1)],
),
axis=0,
)
edge_weights = np.concatenate(
(
np.sum((b - a) * (c - a), axis=1),
np.sum((a - b) * (c - b), axis=1),
np.sum((a - c) * (b - c), axis=1),
)
) / np.tile(cross * 2.0, 3)
edges.sort(axis=1)
edges, inverse = np.unique(edges, axis=0, return_inverse=True)
edge_weights = np.bincount(
inverse,
weights=edge_weights,
minlength=len(edges),
)
interior = np.flatnonzero(~boundary_mask)
field = np.zeros(len(points), dtype=np.float64)
if not len(interior):
return field
compact = np.full(len(points), -1, dtype=np.int64)
compact[interior] = np.arange(len(interior), dtype=np.int64)
# Constrained triangulation can create negative cotangent weights.
# Clamping them keeps the Dirichlet system positive definite.
edge_weights = np.maximum(edge_weights, 0.0)
usable = edge_weights > 1e-14
compact_edges = compact[edges[usable]]
edge_weights = edge_weights[usable]
start_compact = compact_edges[:, 0]
end_compact = compact_edges[:, 1]
start_usable = start_compact >= 0
end_usable = end_compact >= 0
diagonal = np.zeros(len(interior), dtype=np.float64)
diagonal += np.bincount(
start_compact[start_usable],
weights=edge_weights[start_usable],
minlength=len(interior),
)
diagonal += np.bincount(
end_compact[end_usable],
weights=edge_weights[end_usable],
minlength=len(interior),
)
if np.any(diagonal <= 1e-14):
raise ValueError("Unable to build a stable Balloon profile")
interior_edges = start_usable & end_usable
edge_starts = start_compact[interior_edges]
edge_ends = end_compact[interior_edges]
weights = edge_weights[interior_edges]
directed_starts = np.concatenate((edge_starts, edge_ends))
directed_ends = np.concatenate((edge_ends, edge_starts))
directed_weights = np.concatenate((weights, weights))
def multiply(values):
result = diagonal * values
if len(directed_weights):
result -= np.bincount(
directed_starts,
weights=directed_weights * values[directed_ends],
minlength=len(values),
)
return result
right_hand_side = loads[interior]
solution = np.zeros(len(interior), dtype=np.float64)
residual = right_hand_side.copy()
preconditioned = residual / diagonal
direction = preconditioned.copy()
residual_product = float(residual.dot(preconditioned))
target = max(float(np.linalg.norm(right_hand_side)) * tolerance, 1e-12)
max_iterations = max(64, min(len(interior) * 4, 4000))
for _iteration in range(max_iterations):
if float(np.linalg.norm(residual)) <= target:
break
multiplied = multiply(direction)
denominator = float(direction.dot(multiplied))
if denominator <= 0.0 or not math.isfinite(denominator):
raise ValueError("Unable to solve the Balloon profile")
step = residual_product / denominator
solution += step * direction
residual -= step * multiplied
preconditioned = residual / diagonal
next_product = float(residual.dot(preconditioned))
if not math.isfinite(next_product):
raise ValueError("Unable to solve the Balloon profile")
if abs(residual_product) <= 1e-30:
break
direction = preconditioned + next_product / residual_product * direction
residual_product = next_product
field[interior] = np.maximum(solution, 0.0)
return field
def poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation=DEFAULT_INFLATION,
):
import numpy as np
field = cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
)
return 2.0 * float(inflation) * np.sqrt(field)
def balloon_profile(
algorithm,
planar_points,
planar_faces,
boundary_mask,
components,
inflation=DEFAULT_INFLATION,
):
import numpy as np
algorithm = str(algorithm).upper()
if algorithm == "POISSON":
return poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation,
)
if algorithm != "TEDDY":
raise ValueError(f"Unknown Balloon profile algorithm: {algorithm}")
result = np.zeros(len(planar_points), dtype=np.float64)
for start, end, component_faces, outer_contour in components:
result[start:end] = teddy_balloon_heights(
np.asarray(planar_points)[start:end],
component_faces,
outer_contour,
np.asarray(boundary_mask)[start:end],
inflation,
)
return result

169
src/anyimage/operators/cutout_tool/depth_calibration.py Normal file
View File

@@ -0,0 +1,169 @@
"""Calibrate metric reference depth and the Depth Balloon base plane."""
PLANE_VARIANCE_FLOOR_RATIO = 0.025
DEPTH_REFERENCE_PERCENTILE = 95.0
ADAPTIVE_DEPTH_TRANSPARENCY_RATIO = 0.05
DEFAULT_DEPTH_BASELINE = 8.0
DEFAULT_DEPTH_DIRECTION = (0.0, 0.0, 1.0)
def use_adaptive_depth_calibration(selection_transparent_fraction):
"""Return whether the Selection input has enough transparent context."""
return float(selection_transparent_fraction) >= ADAPTIVE_DEPTH_TRANSPARENCY_RATIO
def metric_depth_reference(camera_depth, valid, uniform_scale):
"""Return the shared upper-percentile scene depth for both Depth shapes."""
import numpy as np
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth values and validity mask dimensions do not match")
usable = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
if not usable.any():
raise ValueError("The depth field contains no valid metric depth")
if not uniform_scale > 0.0:
raise ValueError("Depth Uniform Scale must be positive")
reference = np.percentile(
camera_depth[usable],
DEPTH_REFERENCE_PERCENTILE,
)
return float(reference) * float(uniform_scale)
def _sample_valid_bilinear(values, valid, x, y):
import numpy as np
height, width = values.shape
x = np.clip(x, 0.0, max(width - 1, 0))
y = np.clip(y, 0.0, max(height - 1, 0))
x0 = np.floor(x).astype(np.int64)
y0 = np.floor(y).astype(np.int64)
x1 = np.minimum(x0 + 1, width - 1)
y1 = np.minimum(y0 + 1, height - 1)
fx = x - x0
fy = y - y0
samples = np.stack(
(
values[y0, x0],
values[y0, x1],
values[y1, x0],
values[y1, x1],
),
axis=1,
)
weights = np.stack(
(
(1.0 - fx) * (1.0 - fy),
fx * (1.0 - fy),
(1.0 - fx) * fy,
fx * fy,
),
axis=1,
)
sample_valid = np.stack(
(
valid[y0, x0],
valid[y0, x1],
valid[y1, x0],
valid[y1, x1],
),
axis=1,
)
weights *= sample_valid
totals = weights.sum(axis=1)
result = np.zeros(len(x), dtype=np.float32)
usable = totals > 1e-8
result[usable] = (samples[usable] * weights[usable]).sum(axis=1) / totals[usable]
return result, usable
def _weighted_plane(coordinates, values, weights):
import numpy as np
total = float(np.sum(weights))
if total <= 1e-12:
return np.zeros(3, dtype=np.float32)
center = np.sum(coordinates * weights[:, None], axis=0) / total
value_center = float(np.sum(values * weights) / total)
centered = coordinates - center
covariance = (centered * weights[:, None]).T @ centered / total
cross = centered.T @ (weights * (values - value_center)) / total
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
largest = max(float(eigenvalues[-1]), 0.0)
if largest <= 1e-12:
slopes = np.zeros(2, dtype=np.float64)
else:
supported = np.maximum(eigenvalues, largest * PLANE_VARIANCE_FLOOR_RATIO)
slopes = eigenvectors @ ((eigenvectors.T @ cross) / supported)
intercept = value_center - float(center @ slopes)
return np.asarray((*slopes, intercept), dtype=np.float32)
def _stable_plane(coordinates, values):
import numpy as np
coordinates = np.asarray(coordinates, dtype=np.float64)
values = np.asarray(values, dtype=np.float64)
if len(coordinates) < 2:
return np.asarray((0.0, 0.0, np.median(values)), dtype=np.float32)
weights = np.ones(len(values), dtype=np.float64)
plane = np.zeros(3, dtype=np.float32)
for _iteration in range(4):
plane = _weighted_plane(coordinates, values, weights)
residual = values - coordinates @ plane[:2] - plane[2]
centered = residual - np.median(residual)
scale = max(float(np.median(np.abs(centered))) * 1.4826, 1e-6)
weights = np.minimum(
1.0,
(2.5 * scale) / np.maximum(np.abs(centered), 1e-12),
)
return plane
def depth_balloon_projection_parameters(
vertices,
vertex_uv,
camera_depth,
valid,
uniform_scale,
):
"""Fit a metric base plane and return its center depth and direction."""
import numpy as np
vertices = np.asarray(vertices, dtype=np.float32)
uv = np.asarray(vertex_uv, dtype=np.float32)
if vertices.ndim != 2 or vertices.shape[1] != 3:
raise ValueError("Depth Balloon vertices must have three components")
if uv.shape != (len(vertices), 2):
raise ValueError("Depth Balloon UV coordinates do not match its vertices")
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth Balloon depth and mask dimensions do not match")
if not uniform_scale > 0.0:
raise ValueError("Depth Balloon Uniform Scale must be positive")
height, width = camera_depth.shape
sample_x = uv[:, 0] * max(width - 1, 0)
sample_y = (1.0 - uv[:, 1]) * max(height - 1, 0)
calibration_valid = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
sampled, usable = _sample_valid_bilinear(
camera_depth,
calibration_valid,
sample_x,
sample_y,
)
if not usable.any():
raise ValueError("MoGe-2 found no valid Depth Balloon samples")
local_coordinates = vertices[:, :2]
metric_depth = sampled[usable] * float(uniform_scale)
plane = _stable_plane(local_coordinates[usable], metric_depth)
reference = metric_depth_reference(
camera_depth,
calibration_valid,
uniform_scale,
)
direction = np.asarray((plane[0], plane[1], 1.0), dtype=np.float64)
direction /= np.linalg.norm(direction)
return reference, tuple(float(value) for value in direction)

565
src/anyimage/operators/cutout_tool/geometry.py
View File

@@ -5,6 +5,7 @@ from dataclasses import dataclass
from ...preferences import addon_preferences
from ...common.image import image_empty_bounds
from . import balloon
@dataclass(frozen=True)
@@ -661,568 +662,6 @@ def triangulate_selection_contours(
return (*result, components) if return_components else result
DEFAULT_INFLATION = 0.45
def distances_to_contour(points, contour):
import numpy as np
points = np.asarray(points, dtype=np.float64)
contour = np.asarray(contour, dtype=np.float64)
squared = np.full(len(points), np.inf, dtype=np.float64)
for start, end in zip(contour, np.roll(contour, -1, axis=0)):
segment = end - start
length_squared = float(segment.dot(segment))
if length_squared <= 1e-16:
continue
factor = np.clip(
(points - start) @ segment / length_squared,
0.0,
1.0,
)
nearest = start + factor[:, None] * segment
squared = np.minimum(
squared,
np.sum((points - nearest) ** 2, axis=1),
)
return np.sqrt(squared)
def chordal_axis(contour, epsilon=1e-9):
import numpy as np
from mathutils import Vector
from mathutils.geometry import delaunay_2d_cdt
contour = np.asarray(contour, dtype=np.float64)
boundary_count = len(contour)
boundary_edges = [
(index, (index + 1) % boundary_count) for index in range(boundary_count)
]
result = delaunay_2d_cdt(
[Vector(point) for point in contour],
boundary_edges,
[list(range(boundary_count))],
1,
epsilon,
)
points = np.asarray(
[tuple(point) for point in result[0]],
dtype=np.float64,
)
triangles = [tuple(face) for face in result[2] if len(face) == 3]
boundary = {tuple(sorted(edge)) for edge in boundary_edges}
node_indices = {}
nodes = []
segments = set()
def edge_node(edge):
key = ("edge", *tuple(sorted(edge)))
if key not in node_indices:
node_indices[key] = len(nodes)
nodes.append((points[edge[0]] + points[edge[1]]) * 0.5)
return node_indices[key]
for face_index, face in enumerate(triangles):
edges = [
(face[0], face[1]),
(face[1], face[2]),
(face[2], face[0]),
]
internal = [edge for edge in edges if tuple(sorted(edge)) not in boundary]
internal_nodes = [edge_node(edge) for edge in internal]
if len(internal_nodes) == 2:
segments.add(tuple(sorted(internal_nodes)))
elif len(internal_nodes) == 3:
key = ("face", face_index)
center_index = len(nodes)
node_indices[key] = center_index
nodes.append(points[list(face)].mean(axis=0))
for midpoint_index in internal_nodes:
segments.add(tuple(sorted((center_index, midpoint_index))))
if not nodes:
center = contour.mean(axis=0)
return (
np.asarray((center,), dtype=np.float64),
[],
np.asarray(
(distances_to_contour((center,), contour)[0],),
dtype=np.float64,
),
)
nodes = np.asarray(nodes, dtype=np.float64)
radii = distances_to_contour(nodes, contour)
active = set(segments)
while len(active) > 1:
degree = [0] * len(nodes)
for start, end in active:
degree[start] += 1
degree[end] += 1
removable = []
for edge in active:
start, end = edge
leaf = start if degree[start] == 1 else end if degree[end] == 1 else None
if leaf is None:
continue
length = float(np.linalg.norm(nodes[start] - nodes[end]))
if length < max(radii[leaf] * 0.5, epsilon):
removable.append(edge)
if not removable:
break
active.difference_update(removable)
if active:
used = sorted({index for edge in active for index in edge})
remap = {old: new for new, old in enumerate(used)}
nodes = nodes[used]
radii = radii[used]
active = {tuple(sorted((remap[start], remap[end]))) for start, end in active}
else:
widest = int(np.argmax(radii))
nodes = nodes[[widest]]
radii = radii[[widest]]
return nodes, sorted(active), radii
def smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
iterations=2,
factor=0.3,
):
import numpy as np
values = np.asarray(heights, dtype=np.float64).copy()
boundary_mask = np.asarray(boundary_mask, dtype=bool)
neighbors = [set() for _index in range(len(values))]
for face in planar_faces:
for start, end in zip(face, face[1:] + face[:1]):
neighbors[start].add(end)
neighbors[end].add(start)
for _iteration in range(iterations):
updated = values.copy()
for index in np.flatnonzero(~boundary_mask):
adjacent = [
neighbor for neighbor in neighbors[index] if not boundary_mask[neighbor]
]
if adjacent:
average = float(values[adjacent].mean())
updated[index] = values[index] * (1.0 - factor) + average * factor
updated[boundary_mask] = 0.0
values = updated
return values
def teddy_balloon_heights(
planar_points,
planar_faces,
contour,
boundary_mask,
inflation,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
boundary_mask = np.asarray(boundary_mask, dtype=bool)
nodes, segments, radii = chordal_axis(contour)
squared_heights = np.zeros(len(points), dtype=np.float64)
for start, end in segments:
axis_start = nodes[start]
axis_end = nodes[end]
axis = axis_end - axis_start
length_squared = float(axis.dot(axis))
if length_squared <= 1e-16:
continue
factors = np.clip(
(points - axis_start) @ axis / length_squared,
0.0,
1.0,
)
centers = axis_start + factors[:, None] * axis
local_radii = radii[start] + factors * (radii[end] - radii[start])
distance_squared = np.sum((points - centers) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
local_radii * local_radii - distance_squared,
)
degree = np.zeros(len(nodes), dtype=np.int64)
for start, end in segments:
degree[start] += 1
degree[end] += 1
for index in np.flatnonzero(degree != 2):
distance_squared = np.sum((points - nodes[index]) ** 2, axis=1)
squared_heights = np.maximum(
squared_heights,
radii[index] * radii[index] - distance_squared,
)
missing = (~boundary_mask) & (squared_heights <= 0.0)
if np.any(missing):
fallback = distances_to_contour(points[missing], contour)
squared_heights[missing] = fallback * fallback
heights = inflation * np.sqrt(np.maximum(squared_heights, 0.0))
heights[boundary_mask] = 0.0
return smooth_interior_heights(
heights,
planar_faces,
boundary_mask,
)
def cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
tolerance=1e-8,
):
import numpy as np
points = np.asarray(planar_points, dtype=np.float64)
faces = np.asarray(planar_faces, dtype=np.int64)
if faces.size == 0:
faces = np.empty((0, 3), dtype=np.int64)
if faces.ndim != 2 or faces.shape[1] != 3:
raise ValueError("Balloon profile faces must be triangles")
boundary_mask = np.asarray(boundary_mask, dtype=bool)
triangles = points[faces]
a, b, c = (triangles[:, index] for index in range(3))
cross = np.abs(
(b[:, 0] - a[:, 0]) * (c[:, 1] - a[:, 1])
- (b[:, 1] - a[:, 1]) * (c[:, 0] - a[:, 0])
)
usable = cross > 1e-14
faces = faces[usable]
a, b, c = a[usable], b[usable], c[usable]
cross = cross[usable]
loads = np.bincount(
faces.ravel(),
weights=np.repeat(cross / 6.0, 3),
minlength=len(points),
)
edges = np.concatenate(
(
faces[:, (1, 2)],
faces[:, (0, 2)],
faces[:, (0, 1)],
),
axis=0,
)
edge_weights = np.concatenate(
(
np.sum((b - a) * (c - a), axis=1),
np.sum((a - b) * (c - b), axis=1),
np.sum((a - c) * (b - c), axis=1),
)
) / np.tile(cross * 2.0, 3)
edges.sort(axis=1)
edges, inverse = np.unique(edges, axis=0, return_inverse=True)
edge_weights = np.bincount(
inverse,
weights=edge_weights,
minlength=len(edges),
)
interior = np.flatnonzero(~boundary_mask)
field = np.zeros(len(points), dtype=np.float64)
if not len(interior):
return field
compact = np.full(len(points), -1, dtype=np.int64)
compact[interior] = np.arange(len(interior), dtype=np.int64)
# Constrained triangulation can create negative cotangent weights.
# Clamping them keeps the Dirichlet system positive definite.
edge_weights = np.maximum(edge_weights, 0.0)
usable = edge_weights > 1e-14
compact_edges = compact[edges[usable]]
edge_weights = edge_weights[usable]
start_compact = compact_edges[:, 0]
end_compact = compact_edges[:, 1]
start_usable = start_compact >= 0
end_usable = end_compact >= 0
diagonal = np.zeros(len(interior), dtype=np.float64)
diagonal += np.bincount(
start_compact[start_usable],
weights=edge_weights[start_usable],
minlength=len(interior),
)
diagonal += np.bincount(
end_compact[end_usable],
weights=edge_weights[end_usable],
minlength=len(interior),
)
if np.any(diagonal <= 1e-14):
raise ValueError("Unable to build a stable Balloon profile")
interior_edges = start_usable & end_usable
edge_starts = start_compact[interior_edges]
edge_ends = end_compact[interior_edges]
weights = edge_weights[interior_edges]
directed_starts = np.concatenate((edge_starts, edge_ends))
directed_ends = np.concatenate((edge_ends, edge_starts))
directed_weights = np.concatenate((weights, weights))
def multiply(values):
result = diagonal * values
if len(directed_weights):
result -= np.bincount(
directed_starts,
weights=directed_weights * values[directed_ends],
minlength=len(values),
)
return result
right_hand_side = loads[interior]
solution = np.zeros(len(interior), dtype=np.float64)
residual = right_hand_side.copy()
preconditioned = residual / diagonal
direction = preconditioned.copy()
residual_product = float(residual.dot(preconditioned))
target = max(float(np.linalg.norm(right_hand_side)) * tolerance, 1e-12)
max_iterations = max(64, min(len(interior) * 4, 4000))
for _iteration in range(max_iterations):
if float(np.linalg.norm(residual)) <= target:
break
multiplied = multiply(direction)
denominator = float(direction.dot(multiplied))
if denominator <= 0.0 or not math.isfinite(denominator):
raise ValueError("Unable to solve the Balloon profile")
step = residual_product / denominator
solution += step * direction
residual -= step * multiplied
preconditioned = residual / diagonal
next_product = float(residual.dot(preconditioned))
if not math.isfinite(next_product):
raise ValueError("Unable to solve the Balloon profile")
if abs(residual_product) <= 1e-30:
break
direction = preconditioned + next_product / residual_product * direction
residual_product = next_product
field[interior] = np.maximum(solution, 0.0)
return field
def poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation=DEFAULT_INFLATION,
):
import numpy as np
field = cotangent_poisson_field(
planar_points,
planar_faces,
boundary_mask,
)
return 2.0 * float(inflation) * np.sqrt(field)
def balloon_profile(
algorithm,
planar_points,
planar_faces,
boundary_mask,
components,
inflation=DEFAULT_INFLATION,
):
import numpy as np
algorithm = str(algorithm).upper()
if algorithm == "POISSON":
return poisson_balloon_profile(
planar_points,
planar_faces,
boundary_mask,
inflation,
)
if algorithm != "TEDDY":
raise ValueError(f"Unknown Balloon profile algorithm: {algorithm}")
result = np.zeros(len(planar_points), dtype=np.float64)
for start, end, component_faces, outer_contour in components:
result[start:end] = teddy_balloon_heights(
np.asarray(planar_points)[start:end],
component_faces,
outer_contour,
np.asarray(boundary_mask)[start:end],
inflation,
)
return result
PLANE_VARIANCE_FLOOR_RATIO = 0.025
DEPTH_REFERENCE_PERCENTILE = 95.0
ADAPTIVE_DEPTH_TRANSPARENCY_RATIO = 0.05
DEFAULT_DEPTH_BASELINE = 8.0
DEFAULT_DEPTH_DIRECTION = (0.0, 0.0, 1.0)
def use_adaptive_depth_calibration(selection_transparent_fraction):
"""Return whether the Selection input has enough transparent context."""
return float(selection_transparent_fraction) >= ADAPTIVE_DEPTH_TRANSPARENCY_RATIO
def metric_depth_reference(camera_depth, valid, uniform_scale):
"""Return the shared upper-percentile scene depth for both Depth shapes."""
import numpy as np
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth values and validity mask dimensions do not match")
usable = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
if not usable.any():
raise ValueError("The depth field contains no valid metric depth")
if not uniform_scale > 0.0:
raise ValueError("Depth Uniform Scale must be positive")
reference = np.percentile(
camera_depth[usable],
DEPTH_REFERENCE_PERCENTILE,
)
return float(reference) * float(uniform_scale)
def _sample_valid_bilinear(values, valid, x, y):
import numpy as np
height, width = values.shape
x = np.clip(x, 0.0, max(width - 1, 0))
y = np.clip(y, 0.0, max(height - 1, 0))
x0 = np.floor(x).astype(np.int64)
y0 = np.floor(y).astype(np.int64)
x1 = np.minimum(x0 + 1, width - 1)
y1 = np.minimum(y0 + 1, height - 1)
fx = x - x0
fy = y - y0
samples = np.stack(
(
values[y0, x0],
values[y0, x1],
values[y1, x0],
values[y1, x1],
),
axis=1,
)
weights = np.stack(
(
(1.0 - fx) * (1.0 - fy),
fx * (1.0 - fy),
(1.0 - fx) * fy,
fx * fy,
),
axis=1,
)
sample_valid = np.stack(
(
valid[y0, x0],
valid[y0, x1],
valid[y1, x0],
valid[y1, x1],
),
axis=1,
)
weights *= sample_valid
totals = weights.sum(axis=1)
result = np.zeros(len(x), dtype=np.float32)
usable = totals > 1e-8
result[usable] = (samples[usable] * weights[usable]).sum(axis=1) / totals[usable]
return result, usable
def _weighted_plane(coordinates, values, weights):
import numpy as np
total = float(np.sum(weights))
if total <= 1e-12:
return np.zeros(3, dtype=np.float32)
center = np.sum(coordinates * weights[:, None], axis=0) / total
value_center = float(np.sum(values * weights) / total)
centered = coordinates - center
covariance = (centered * weights[:, None]).T @ centered / total
cross = centered.T @ (weights * (values - value_center)) / total
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
largest = max(float(eigenvalues[-1]), 0.0)
if largest <= 1e-12:
slopes = np.zeros(2, dtype=np.float64)
else:
supported = np.maximum(eigenvalues, largest * PLANE_VARIANCE_FLOOR_RATIO)
slopes = eigenvectors @ ((eigenvectors.T @ cross) / supported)
intercept = value_center - float(center @ slopes)
return np.asarray((*slopes, intercept), dtype=np.float32)
def _stable_plane(coordinates, values):
import numpy as np
coordinates = np.asarray(coordinates, dtype=np.float64)
values = np.asarray(values, dtype=np.float64)
if len(coordinates) < 2:
return np.asarray((0.0, 0.0, np.median(values)), dtype=np.float32)
weights = np.ones(len(values), dtype=np.float64)
plane = np.zeros(3, dtype=np.float32)
for _iteration in range(4):
plane = _weighted_plane(coordinates, values, weights)
residual = values - coordinates @ plane[:2] - plane[2]
centered = residual - np.median(residual)
scale = max(float(np.median(np.abs(centered))) * 1.4826, 1e-6)
weights = np.minimum(
1.0,
(2.5 * scale) / np.maximum(np.abs(centered), 1e-12),
)
return plane
def depth_balloon_projection_parameters(
vertices,
vertex_uv,
camera_depth,
valid,
uniform_scale,
):
"""Fit a metric base plane and return its center depth and direction."""
import numpy as np
vertices = np.asarray(vertices, dtype=np.float32)
uv = np.asarray(vertex_uv, dtype=np.float32)
if vertices.ndim != 2 or vertices.shape[1] != 3:
raise ValueError("Depth Balloon vertices must have three components")
if uv.shape != (len(vertices), 2):
raise ValueError("Depth Balloon UV coordinates do not match its vertices")
camera_depth = np.asarray(camera_depth, dtype=np.float32)
valid = np.asarray(valid, dtype=bool)
if valid.shape != camera_depth.shape:
raise ValueError("Depth Balloon depth and mask dimensions do not match")
if not uniform_scale > 0.0:
raise ValueError("Depth Balloon Uniform Scale must be positive")
height, width = camera_depth.shape
sample_x = uv[:, 0] * max(width - 1, 0)
sample_y = (1.0 - uv[:, 1]) * max(height - 1, 0)
calibration_valid = valid & np.isfinite(camera_depth) & (camera_depth > 0.0)
sampled, usable = _sample_valid_bilinear(
camera_depth,
calibration_valid,
sample_x,
sample_y,
)
if not usable.any():
raise ValueError("MoGe-2 found no valid Depth Balloon samples")
local_coordinates = vertices[:, :2]
metric_depth = sampled[usable] * float(uniform_scale)
plane = _stable_plane(local_coordinates[usable], metric_depth)
reference = metric_depth_reference(
camera_depth,
calibration_valid,
uniform_scale,
)
direction = np.asarray((plane[0], plane[1], 1.0), dtype=np.float64)
direction /= np.linalg.norm(direction)
return reference, tuple(float(value) for value in direction)
CUTOUT_ALPHA_THRESHOLD = 0.9
CUTOUT_CLEANUP_UI_SCALE = 100.0
@@ -1313,7 +752,7 @@ def cutout_domain(
planar = np.concatenate(refined_points)
planar_faces, components = refined_faces, refined_components
boundary = boundary_vertices_from_faces(len(planar), planar_faces)
profile = balloon_profile(
profile = balloon.balloon_profile(
balloon_algorithm, planar, planar_faces, boundary, components
)
faces = np.asarray(planar_faces, dtype=np.int32)

9
src/anyimage/operators/cutout_tool/interaction.py
View File

@@ -31,9 +31,9 @@ def _pie_button(
selection_path_json,
generate_normal,
point_spacing,
inward_axis,
alpha_threshold,
fine_outline,
gesture,
):
operator = pie.operator(
"anyimage.cutout_selection_to_shape",
@@ -45,9 +45,9 @@ def _pie_button(
operator.shape = shape
operator.generate_normal = generate_normal
operator.point_spacing = point_spacing
operator.inward_axis = inward_axis
operator.alpha_threshold = alpha_threshold
operator.fine_outline = fine_outline
operator.gesture = gesture
def open_shape_pie(
@@ -55,6 +55,8 @@ def open_shape_pie(
event,
source_object,
selection_path,
*,
gesture="LASSO",
):
source_name = source_object.name
selection_path_json = selection_path.to_json()
@@ -64,7 +66,6 @@ def open_shape_pie(
generate_normal = ready and bool(
getattr(settings, "cutout_generate_normal", False)
)
inward_axis = str(getattr(settings, "cutout_inward_axis", "NEGATIVE_Z"))
alpha_threshold = settings.cutout_alpha_threshold
fine_outline = settings.cutout_fine_outline
@@ -81,9 +82,9 @@ def open_shape_pie(
selection_path_json,
generate_normal,
point_spacing,
inward_axis,
alpha_threshold,
fine_outline,
gesture,
)
context.window_manager.popup_menu_pie(

71
src/anyimage/operators/cutout_tool/object.py
View File

@@ -15,23 +15,24 @@ from ...common.object import (
set_modifier_input,
)
from ...common.node import load_node_group
from .geometry import (
CUTOUT_ALPHA_THRESHOLD,
from .depth_calibration import (
DEFAULT_DEPTH_BASELINE,
DEFAULT_DEPTH_DIRECTION,
depth_balloon_projection_parameters,
metric_depth_reference,
use_adaptive_depth_calibration,
)
from .geometry import (
CUTOUT_ALPHA_THRESHOLD,
crop_plane_bounds,
cutout_cleanup_value,
cutout_sampling_spacing,
depth_balloon_projection_parameters,
metric_depth_reference,
selection_calibration,
use_adaptive_depth_calibration,
world_plane_metric,
)
from .shape import (
BALLOON_ATTRIBUTE_NAME,
NORMAL_SCALE_ATTRIBUTE_NAME,
CUTOUT_X_TO_Z,
CUTOUT_NODE_GROUP_NAMES,
DEPTH_CUTOUT_SHAPES,
)
@@ -43,38 +44,11 @@ class NormalMode(StrEnum):
OBJECT = "OBJECT"
class CutoutInwardAxis(StrEnum):
NEGATIVE_Z = "NEGATIVE_Z"
POSITIVE_X = "POSITIVE_X"
def _cutout_world_matrix(source_matrix, crop_center, inward_axis):
def _cutout_world_matrix(source_matrix, crop_center):
from mathutils import Matrix
center_x, center_y = (float(value) for value in crop_center)
matrix = source_matrix @ Matrix.Translation((center_x, center_y, 0.0))
if CutoutInwardAxis(inward_axis) == CutoutInwardAxis.POSITIVE_X:
matrix @= Matrix(CUTOUT_X_TO_Z)
return matrix
def orient_object_normal_image(image, inward_axis):
"""Convert native Z-axis Cutout normals to the selected object axes."""
if CutoutInwardAxis(inward_axis) == CutoutInwardAxis.NEGATIVE_Z:
return image
import numpy as np
pixels = np.empty(int(image.size[0]) * int(image.size[1]) * 4, dtype=np.float32)
image.pixels.foreach_get(pixels)
rgba = pixels.reshape((-1, 4))
normal = rgba[:, :3] * 2.0 - 1.0
rgba[:, :3] = np.column_stack(
(-normal[:, 2], -normal[:, 0], normal[:, 1])
) * 0.5 + 0.5
image.pixels.foreach_set(pixels)
image.update()
image.pack()
return image
return source_matrix @ Matrix.Translation((center_x, center_y, 0.0))
def _assign_uv(mesh, uv):
@@ -99,7 +73,7 @@ def _assign_float_point_attribute(mesh, name, values):
def load_cutout_node_group(shape):
return load_node_group(CUTOUT_NODE_GROUP_NAMES[shape], asset_revision=23 if shape == "DEPTH_SOLID" else 12 if shape == "DEPTH_BALLOON" else 11)
return load_node_group(CUTOUT_NODE_GROUP_NAMES[shape], asset_revision=26 if shape == "DEPTH_SOLID" else 14 if shape == "DEPTH_BALLOON" else 13)
def _connect_cutout_normal_scale(material):
@@ -134,13 +108,12 @@ def create_shape_object(
depth_metadata=None,
normal_image=None,
normal_mode=NormalMode.NONE,
inward_axis=CutoutInwardAxis.NEGATIVE_Z,
color_image,
gesture="LASSO",
):
import numpy as np
normal_mode = NormalMode(normal_mode)
inward_axis = CutoutInwardAxis(inward_axis)
bounds = crop_plane_bounds(
source_object,
content_bounds,
@@ -229,7 +202,6 @@ def create_shape_object(
mesh_object.matrix_world = _cutout_world_matrix(
source_object.matrix_world,
crop_center,
inward_axis,
)
mesh_object["anyimage_mesh_shape"] = shape
node_group = load_cutout_node_group(shape)
@@ -237,10 +209,6 @@ def create_shape_object(
modifier.node_group = node_group
modifier.show_group_selector = False
for name, value in (
(
"Inward Axis",
0 if inward_axis == CutoutInwardAxis.NEGATIVE_Z else 1,
),
("Cleanup Threshold", cleanup_value),
(
"Thickness",
@@ -249,9 +217,24 @@ def create_shape_object(
):
set_modifier_input(
modifier,
modifier_input_identifier(node_group, name),
modifier_input_identifier(
node_group, name,
subtype="NONE" if name == "Thickness" and shape == "DEPTH_SOLID" else None,
),
value,
)
if shape in {"FLAT", "SOLID", "DEPTH_SOLID"} and gesture == "POLYLINE":
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Mode"),
1,
)
if shape == "SOLID":
set_modifier_input(
modifier,
modifier_input_identifier(node_group, "Thickness", subtype="DISTANCE"),
0.5,
)
if shape in DEPTH_CUTOUT_SHAPES:
for name, value in (
("Depth Scale", 1.0),

79
src/anyimage/operators/cutout_tool/operators.py
View File

@@ -13,6 +13,7 @@ from ...preferences import (
from ...properties import (
cutout_alpha_threshold_property,
cutout_fine_outline_property,
cutout_gesture_property,
)
from ...runtime import JobOperatorBase
from ...server.model_catalog import DEFAULT_GEOMETRY_MODEL_KEY
@@ -35,7 +36,7 @@ from ...common.image import (
image_rgba,
load_normal_result_image,
require_image_empty,
video_total_frames,
is_animated_image,
)
from ...common.depth import load_depth_metadata, load_depth_result_image
from ...common.color_image import (
@@ -47,7 +48,7 @@ from ...common.color_image import (
)
from ...common.viewport import (
ImageGesture,
image_edit_drag_keymap,
image_edit_point_keymap,
report_image_edit_exception,
)
from ...common.selection import (
@@ -62,10 +63,8 @@ from .geometry import (
)
from .interaction import open_shape_pie
from .object import (
CutoutInwardAxis,
NormalMode,
create_shape_object,
orient_object_normal_image,
)
from .shape import CUTOUT_SHAPES, DEPTH_CUTOUT_SHAPES
@@ -100,8 +99,8 @@ def create_cutout_shape(
depth_metadata=None,
normal_image=None,
normal_mode=NormalMode.NONE,
inward_axis=CutoutInwardAxis.NEGATIVE_Z,
color_image,
gesture="LASSO",
):
base_shape = build_base_shape(
context,
@@ -125,8 +124,8 @@ def create_cutout_shape(
depth_metadata=depth_metadata,
normal_image=normal_image,
normal_mode=normal_mode,
inward_axis=inward_axis,
color_image=color_image,
gesture=gesture,
)
undo_message = f"Convert to {shape.replace('_', ' ').title()}"
if "FINISHED" not in bpy.ops.ed.undo_push(message=undo_message):
@@ -191,6 +190,7 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
selection_path_json: bpy.props.StringProperty(options=_HIDDEN)
selection_bounds_json: bpy.props.StringProperty(options=_HIDDEN)
shape: bpy.props.StringProperty(default="SOLID", options=_HIDDEN)
gesture: bpy.props.StringProperty(default="LASSO", options=_HIDDEN)
point_spacing: bpy.props.FloatProperty(
default=32.0,
min=0.01,
@@ -215,23 +215,13 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
generate_normal: bpy.props.BoolProperty(default=False, options=_HIDDEN)
alpha_threshold: cutout_alpha_threshold_property()
fine_outline: cutout_fine_outline_property()
inward_axis: bpy.props.StringProperty(
default=CutoutInwardAxis.NEGATIVE_Z.value,
options=_HIDDEN,
)
def _needs_generation(self):
return self.shape in DEPTH_CUTOUT_SHAPES or self.generate_normal
def execute(self, context):
try:
self.inward_axis = getattr(
self,
"inward_axis",
CutoutInwardAxis.NEGATIVE_Z.value,
)
source_object = require_image_empty(self.source_object_name)
if video_total_frames(source_object.data) != 1:
if is_animated_image(source_object.data):
raise ValueError("Cutout currently supports still images")
if self.shape not in CUTOUT_SHAPES:
raise ValueError(f"Unsupported Cutout shape: {self.shape}")
@@ -263,8 +253,9 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
create_cutout_shape(
context, source_object, self.shape, self.point_spacing,
content_values, selection_mask.bounds,
inward_axis=self.inward_axis, alpha_threshold=self.alpha_threshold,
alpha_threshold=self.alpha_threshold,
fine_outline=self.fine_outline, color_image=color_image,
gesture=self.gesture,
)
return {"FINISHED"}
setup_result = invoke_ai_setup_if_needed(self)
@@ -317,21 +308,14 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
)
parameters.update(
moge2_parameters(
context.scene.anyimage_settings,
source_path,
self.model,
self.resolution_level,
1,
)
)
return parameters
def response(self, context, result):
inward_axis = getattr(
self,
"inward_axis",
CutoutInwardAxis.NEGATIVE_Z.value,
)
source_object = require_image_empty(self.source_object_name)
source_bounds = tuple(json.loads(self.selection_bounds_json))
color_image, source_alpha, color_bounds = _generated_color_result(
@@ -358,8 +342,6 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
result.file(normal_key),
source_object,
)
if normal_mode == NormalMode.OBJECT:
orient_object_normal_image(normal_image, inward_axis)
depth_metadata = None
if shape_uses_depth(self.shape):
depth_metadata = load_depth_metadata(result.file("depth_metadata"))
@@ -379,10 +361,10 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
depth_metadata=depth_metadata,
normal_image=normal_image,
normal_mode=normal_mode,
inward_axis=inward_axis,
alpha_threshold=self.alpha_threshold,
fine_outline=self.fine_outline,
color_image=color_image,
gesture=self.gesture,
)
except Exception:
if normal_image is not None and normal_image.users == 0:
@@ -401,22 +383,32 @@ class CutoutSelectionToShape(JobOperatorBase, bpy.types.Operator):
class SelectCutoutSelection(ImageGesture, bpy.types.Operator):
bl_idname = "anyimage.select_cutout_selection"
bl_label = "Start Cutout Selection"
bl_description = "Lasso an Image Empty selection, then choose its mesh shape"
bl_description = "Select an Image Empty area with a lasso or polyline, then choose its mesh shape"
bl_options = {"INTERNAL"}
active_tool_id = "anyimage.cutout_lasso"
gesture = "LASSO"
resolve_on_press = True
gesture: cutout_gesture_property()
source_object_name: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
source_matrix_data: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
_pie_event = None
_polyline_confirm_type = None
def _read_gesture_settings(self, context):
self.gesture = context.scene.anyimage_settings.cutout_gesture
def _set_status(self, context):
action = (
"click points; click start, double-click, or Enter for shapes; Backspace removes"
if self.gesture == "POLYLINE" else "drag and release for shapes"
)
context.workspace.status_text_set(
"Cutout Lasso: drag and release for shapes; RMB or Esc cancels"
f"Cutout {self.gesture.title()}: {action}; RMB or Esc cancels"
)
def invoke(self, context, event):
self._polyline_confirm_type = None
self._pie_event = event
return super().invoke(context, event)
@@ -424,6 +416,19 @@ class SelectCutoutSelection(ImageGesture, bpy.types.Operator):
self._pie_event = event
return ImageGesture.modal(self, context, event)
def _confirm_polyline(self, context):
if len(self._path) >= 3:
self._polyline_confirm_type = self._pie_event.type
return {"RUNNING_MODAL"}
def _modal_polyline(self, context, event):
if self._polyline_confirm_type is not None:
if event.type == self._polyline_confirm_type and event.value == "RELEASE":
self._polyline_confirm_type = None
return ImageGesture._confirm_polyline(self, context)
return {"RUNNING_MODAL"}
return ImageGesture._modal_polyline(self, context, event)
def _submit_selection_path(self, context, selection_path):
try:
source_object = require_image_empty(self.source_object_name)
@@ -435,6 +440,7 @@ class SelectCutoutSelection(ImageGesture, bpy.types.Operator):
self._pie_event,
source_object,
selection_path,
gesture=self.gesture,
)
return {"FINISHED"}
@@ -444,19 +450,18 @@ class CutoutTool(bpy.types.WorkSpaceTool):
bl_context_mode = "OBJECT"
bl_idname = "anyimage.cutout_lasso"
bl_label = "Cutout"
bl_description = "Lasso an Image Empty and choose a mesh shape"
bl_description = "Select an Image Empty area with a lasso or polyline and choose a mesh shape"
bl_icon = "ops.mesh.primitive_sphere_add_gizmo"
bl_operator = SelectCutoutSelection.bl_idname
bl_keymap = image_edit_drag_keymap(
bl_keymap = image_edit_point_keymap(
SelectCutoutSelection.bl_idname,
deselect_on_empty=False,
)
def draw_settings(context, layout, _tool):
settings = context.scene.anyimage_settings
status = draw_ai_setup(layout)
draw_ai_property(layout, settings, "cutout_generate_normal", status)
layout.prop(settings, "cutout_gesture")
layout.prop(settings, "cutout_mesh_detail")
layout.prop(settings, "cutout_fine_outline")
layout.prop(settings, "cutout_alpha_threshold")
layout.prop(settings, "cutout_inward_axis")
status = draw_ai_setup(layout)
draw_ai_property(layout, settings, "cutout_generate_normal", status)

6
src/anyimage/operators/cutout_tool/shape.py
View File

@@ -2,12 +2,6 @@
BALLOON_ATTRIBUTE_NAME = "o_balloon"
NORMAL_SCALE_ATTRIBUTE_NAME = "o_normal_scale"
CUTOUT_X_TO_Z = (
(0.0, -1.0, 0.0, 0.0),
(0.0, 0.0, 1.0, 0.0),
(-1.0, 0.0, 0.0, 0.0),
(0.0, 0.0, 0.0, 1.0),
)
CUTOUT_Z_TO_X = (
(0.0, 0.0, -1.0, 0.0),
(-1.0, 0.0, 0.0, 0.0),

648
src/anyimage/operators/frame_tool.py
View File

@@ -1,648 +0,0 @@
import math
import bpy
from ..common.image import (
MAX_PROJECTIVE_OUTPUT_PIXELS,
PROJECTIVE_CHUNK_ROWS,
create_image_edit_result,
homography_from_points,
image_empty_bounds,
image_pixels,
is_image_empty,
premultiplied_rgba,
replace_empty_image,
sample_rgba,
transform_homography,
video_total_frames,
)
from ..preferences import configured_max_frame_resolution
from ..common.viewport import (
active_image_edit_source,
active_view3d_tool_id,
draw_dashed_line,
draw_polygon_fill,
drawing_in_region,
image_edit_drag_keymap,
image_edit_poll,
rectangle_screen_path,
report_image_edit_exception,
)
from ..common.selection import ImageEditWarning, clip_polygon_halfplanes
FRAME_TOOL_ID = "anyimage.frame"
PROJECTIVE_RELATIVE_EPSILON = 1e-12
def _matrix_array(matrix):
import numpy as np
return np.asarray(
tuple(tuple(float(value) for value in row) for row in matrix),
dtype=np.float64,
)
def frame_source_projection(
projection_matrix,
view_matrix,
matrix_world,
bounds,
region_size,
*,
perspective,
):
"""Return normalized image-plane transforms for one frozen view."""
import numpy as np
x_min, x_max, y_min, y_max = (float(value) for value in bounds)
region_width, region_height = (float(value) for value in region_size)
if min(x_max - x_min, y_max - y_min, region_width, region_height) <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_plane = np.asarray(
(
(x_max - x_min, 0.0, x_min),
(0.0, y_min - y_max, y_max),
(0.0, 0.0, 0.0),
(0.0, 0.0, 1.0),
),
dtype=np.float64,
)
world = _matrix_array(matrix_world)
source_to_world = world @ source_plane
source_to_view = _matrix_array(view_matrix) @ source_to_world
source_to_clip = _matrix_array(projection_matrix) @ source_to_world
source_to_screen = np.stack(
(
0.5 * (source_to_clip[0] + source_to_clip[3]),
0.5 * (source_to_clip[1] + source_to_clip[3]),
source_to_clip[3],
)
)
scale = float(np.max(np.abs(source_to_screen)))
if not math.isfinite(scale) or scale <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_to_screen /= scale
if not np.isfinite(source_to_screen).all():
raise ValueError("The Image Empty projection is invalid")
condition = float(np.linalg.cond(source_to_screen))
if not math.isfinite(condition) or condition > 1.0 / np.finfo(np.float64).eps:
raise ValueError("The Image Empty is edge-on to the current view")
return {
"bounds": (x_min, x_max, y_min, y_max),
"origin_depth": -float((_matrix_array(view_matrix) @ world)[2, 3]),
"perspective": bool(perspective),
"region_size": (region_width, region_height),
"screen_to_source": np.linalg.inv(source_to_screen),
"source_to_screen": source_to_screen,
"source_to_view": source_to_view,
"source_to_world": source_to_world,
}
def _projective_points(points, matrix):
"""Map points and return finite results plus per-point validity."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
flat = points.reshape((-1, 2))
homogeneous = np.column_stack((flat, np.ones(len(flat), dtype=np.float64)))
mapped = homogeneous @ np.asarray(matrix, dtype=np.float64).T
magnitude = np.max(np.abs(mapped), axis=1)
valid = (
np.isfinite(mapped).all(axis=1)
& (np.abs(mapped[:, 2]) > PROJECTIVE_RELATIVE_EPSILON * magnitude)
)
result = np.full((len(flat), 2), np.nan, dtype=np.float64)
result[valid] = mapped[valid, :2] / mapped[valid, 2, None]
return result.reshape(points.shape), valid.reshape(points.shape[:-1])
def frame_source_coordinates(source, screen_points):
"""Return source pixels, view depth, validity, and normalized coordinates."""
import numpy as np
screen_points = np.asarray(screen_points, dtype=np.float64)
region_width, region_height = source["region_size"]
normalized_screen = screen_points / (region_width, region_height)
normalized_source, valid = _projective_points(
normalized_screen,
source["screen_to_source"],
)
flat_source = normalized_source.reshape((-1, 2))
homogeneous = np.column_stack(
(flat_source, np.ones(len(flat_source), dtype=np.float64))
)
depth = -(homogeneous @ source["source_to_view"][2])
depth = depth.reshape(valid.shape)
valid &= np.isfinite(depth)
if source["perspective"]:
valid &= depth > 0.0
valid &= (
(normalized_source[..., 0] >= 0.0)
& (normalized_source[..., 0] <= 1.0)
& (normalized_source[..., 1] >= 0.0)
& (normalized_source[..., 1] <= 1.0)
)
image_width, image_height = source["image_size"]
source_points = normalized_source * (image_width, image_height)
source_points[~valid] = -1.0
return source_points, depth, valid, normalized_source
def frame_source_overlap(source, frame_quad):
"""Clip the source plane before dividing its homogeneous projection."""
import numpy as np
frame = np.asarray(frame_quad, dtype=np.float64) / source["region_size"]
left, bottom = frame.min(axis=0)
right, top = frame.max(axis=0)
x, y, w = source["source_to_screen"]
boundaries = [x - left * w, right * w - x, y - bottom * w, top * w - y]
if source["perspective"]:
boundaries.insert(0, -source["source_to_view"][2])
polygon = clip_polygon_halfplanes(
((0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0)), boundaries
)
if len(polygon) < 3:
return ()
screen, valid = _projective_points(polygon, source["source_to_screen"])
if not np.all(valid):
return ()
screen *= source["region_size"]
# Center the coordinates to avoid cancellation in a distant small polygon.
centered = screen - screen[0]
area = abs(np.sum(centered[:, 0] * np.roll(centered[:, 1], -1)
- centered[:, 1] * np.roll(centered[:, 0], -1))) * 0.5
return tuple(map(tuple, screen)) if area > 1e-8 else ()
def polygon_centroid(polygon):
"""Return the area centroid of one non-degenerate screen polygon."""
cross_sum = 0.0
x_sum = 0.0
y_sum = 0.0
for first, second in zip(polygon, (*polygon[1:], polygon[0])):
cross = first[0] * second[1] - second[0] * first[1]
cross_sum += cross
x_sum += (first[0] + second[0]) * cross
y_sum += (first[1] + second[1]) * cross
if abs(cross_sum) <= 1e-12:
raise ValueError("Frame does not overlap the active Image Empty")
return x_sum / (3.0 * cross_sum), y_sum / (3.0 * cross_sum)
def _pixel_quad(image_size):
width, height = (float(value) for value in image_size)
if min(width, height) <= 0.0:
raise ValueError("The source image has no readable dimensions")
return ((0.0, 0.0), (width, 0.0), (width, height), (0.0, height))
def frame_output_size(frame_quad, region_size, max_resolution):
"""Choose output resolution from Frame pixels within the current Region."""
frame_width = abs(float(frame_quad[1][0] - frame_quad[0][0]))
frame_height = abs(float(frame_quad[0][1] - frame_quad[3][1]))
region_width, region_height = (float(value) for value in region_size)
maximum = float(max_resolution)
if min(frame_width, frame_height, region_width, region_height, maximum) <= 0.0:
raise ValueError("The Frame output dimensions must be positive")
scale = min(
1.0,
region_width / frame_width,
region_height / frame_height,
maximum / max(frame_width, frame_height),
math.sqrt(
MAX_PROJECTIVE_OUTPUT_PIXELS / max(frame_width * frame_height, 1.0)
),
)
return (
max(2, int(round(frame_width * scale))),
max(2, int(round(frame_height * scale))),
)
def _frame_source(source_object, region, view3d, view_matrix, *, active):
import numpy as np
image_size = tuple(source_object.data.size)
pixels = image_pixels(source_object.data)
width, height = image_size
bounds = image_empty_bounds(source_object)
source = frame_source_projection(
view3d.perspective_matrix,
view_matrix,
source_object.matrix_world,
bounds,
(region.width, region.height),
perspective=view3d.is_perspective,
)
source.update(
{
"active": bool(active),
"image_size": image_size,
"name": source_object.name,
"object": source_object,
"pixels": premultiplied_rgba(pixels, image_size),
"rgba": np.flipud(pixels.reshape((height, width, 4))),
}
)
return source
def _composite_frame_samples(sources, screen_points):
"""Resolve independently sorted samples with an unweighted bottom RGB base."""
import numpy as np
samples, depths, coordinates = [], [], []
for source in sources:
points, depth, valid, _ = frame_source_coordinates(source, screen_points)
sample = sample_rgba(source["pixels"], points)
sample[~valid] = 0
depths.append(np.where(valid, depth, -math.inf))
samples.append(sample)
coordinates.append(points)
samples = np.stack(samples)
depths = np.stack(depths)
order = np.argsort(-depths, axis=0, kind="stable")
bottom = order[0]
covered = np.isfinite(np.take_along_axis(depths, bottom[None], axis=0)[0])
result = np.zeros_like(samples[0])
for index, source in enumerate(sources):
selected = covered & (bottom == index)
if np.any(selected):
result[..., :3][selected] = sample_rgba(
source["rgba"], coordinates[index][selected]
)[..., :3]
for index in range(len(sources)):
layer = np.take_along_axis(samples, order[index][None, ..., None], axis=0)[0]
alpha = layer[..., 3:4]
result[..., 3:4] = alpha + result[..., 3:4] * (1 - alpha)
if index:
result[..., :3] = layer[..., :3] + result[..., :3] * (1 - alpha)
return result, covered
def _frame_boundary_pixels(sources, points, output_to_screen):
"""Conservatively detect projected contour and depth-order crossings."""
import numpy as np
boundary = np.zeros(points.shape[:-1], dtype=bool)
for source in sources:
width, height = source["region_size"]
mapping = source["screen_to_source"] @ np.diag((1 / width, 1 / height, 1)) @ output_to_screen
u, v, w = mapping
lines = [u, v, u - w, v - w, w]
positive = np.ones_like(boundary)
negative = np.ones_like(boundary)
for line in (u, v, w - u, w - v, w):
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
positive &= value + radius >= 0
negative &= value - radius <= 0
source_boundary = np.zeros_like(boundary)
if source["perspective"]:
lines.append(source["source_to_view"][2] @ mapping)
for line in lines:
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
source_boundary |= np.abs(value) < radius - 1e-10 * max(radius, 1e-20)
boundary |= source_boundary & (positive | negative)
if len(sources) > 1:
first_order = None
for offset in ((-0.499999, -0.499999), (0.499999, -0.499999),
(-0.499999, 0.499999), (0.499999, 0.499999)):
screen = transform_homography(points + offset, output_to_screen)
depths = []
for source in sources:
_, depth, valid, _ = frame_source_coordinates(source, screen)
depths.append(np.where(valid, depth, -math.inf))
order = np.argsort(-np.stack(depths), axis=0, kind="stable")
if first_order is None:
first_order = order
else:
boundary |= np.any(order != first_order, axis=0)
return boundary
def composite_frame_pixels(sources, frame_quad, output_size):
"""Composite projected Image Empty sources into one Blender-order RGBA image."""
import numpy as np
output_width, output_height = (int(value) for value in output_size)
if min(output_width, output_height) <= 0:
raise ValueError("The Frame output dimensions must be positive")
sources = tuple(
sorted(sources, key=lambda source: (source["active"], source["name"]))
)
if not sources or not sources[-1]["active"]:
raise ValueError("Frame requires one active Image Empty")
output_quad = _pixel_quad(output_size)
output_to_screen = homography_from_points(output_quad, frame_quad)
output = np.zeros((output_height, output_width, 4), dtype=np.float32)
covered = np.zeros((output_height, output_width), dtype=bool)
target_x = np.arange(output_width, dtype=np.float64) + 0.5
for row_start in range(0, output_height, PROJECTIVE_CHUNK_ROWS):
row_end = min(row_start + PROJECTIVE_CHUNK_ROWS, output_height)
target_y = np.arange(row_start, row_end, dtype=np.float64) + 0.5
x, y = np.meshgrid(target_x, target_y)
points = np.stack((x, y), axis=-1)
composite, coverage = _composite_frame_samples(
sources, transform_homography(points, output_to_screen)
)
boundary = _frame_boundary_pixels(sources, points, output_to_screen)
if np.any(boundary):
edge_points = points[boundary]
accumulated = np.zeros((len(edge_points), 4), dtype=np.float32)
counts = np.zeros(len(edge_points), dtype=np.int32)
for dy in (-0.375, -0.125, 0.125, 0.375):
for dx in (-0.375, -0.125, 0.125, 0.375):
sample, hit = _composite_frame_samples(
sources,
transform_homography(edge_points + (dx, dy), output_to_screen),
)
accumulated += sample
counts += hit
accumulated[..., 3] /= 16
accumulated[..., :3] /= np.maximum(counts, 1)[..., None]
composite[boundary] = accumulated
coverage[boundary] = counts > 0
covered[row_start:row_end] = coverage
output[row_start:row_end] = composite
output[..., 3][output[..., 3] <= 1e-8] = 0.0
output[~covered] = 0.0
return np.flipud(output).ravel()
def view_facing_frame_geometry(corners, output_size):
"""Return a world matrix array and display size for frame-plane corners."""
import numpy as np
corners = np.asarray(corners, dtype=np.float64)
if corners.shape != (4, 3) or not np.isfinite(corners).all():
raise ValueError("The Frame view plane is invalid")
top_left, top_right, _bottom_right, bottom_left = corners
right = top_right - top_left
up = top_left - bottom_left
width = float(np.linalg.norm(right))
height = float(np.linalg.norm(up))
if min(width, height) <= 1e-10:
raise ValueError("Draw a larger Frame")
right /= width
up /= height
forward = np.cross(right, up)
forward_length = float(np.linalg.norm(forward))
if forward_length <= 1e-10:
raise ValueError("The Frame view plane is invalid")
forward /= forward_length
matrix = np.identity(4, dtype=np.float64)
matrix[:3, 0] = right
matrix[:3, 1] = up
matrix[:3, 2] = forward
matrix[:3, 3] = corners.mean(axis=0)
display_size = width if output_size[0] >= output_size[1] else height
return matrix, display_size
def frame_result_transform(frame_quad, source_origin, region, view3d, output_size):
"""Return the view-facing object matrix and Image Empty display size."""
from mathutils import Matrix
from bpy_extras import view3d_utils
corners = tuple(
tuple(
float(value)
for value in view3d_utils.region_2d_to_location_3d(
region,
view3d,
point,
source_origin,
)
)
for point in frame_quad
)
matrix, display_size = view_facing_frame_geometry(corners, output_size)
return Matrix(tuple(tuple(row) for row in matrix)), display_size
def frame_depth_location(active_source, overlap_polygon):
"""Return the world point whose view depth places the Frame result."""
import numpy as np
source_object = active_source["object"]
if not active_source["perspective"] or active_source["origin_depth"] > 0.0:
return source_object.matrix_world.translation.copy()
centroid = np.asarray((polygon_centroid(overlap_polygon),), dtype=np.float64)
_points, depth, valid, normalized_source = frame_source_coordinates(
active_source,
centroid,
)
if not bool(valid[0]) or depth[0] <= 0.0:
raise ValueError("The Frame result depth is invalid")
source_point = np.asarray(
(normalized_source[0, 0], normalized_source[0, 1], 1.0),
dtype=np.float64,
)
world = active_source["source_to_world"] @ source_point
if not np.isfinite(world).all() or abs(world[3]) <= 1e-12:
raise ValueError("The Frame result depth is invalid")
return tuple(float(value) for value in world[:3] / world[3])
def selected_frame_sources(context):
"""Return selected still Image Empties with the active object last."""
active = active_image_edit_source(context)
sources = [
source for source in context.selected_objects if is_image_empty(source)
]
for source in sources:
if video_total_frames(source.data) > 1:
raise ValueError("Frame only supports still images")
return tuple(
sorted(sources, key=lambda source: (source is active, source.name))
)
def apply_frame(context, source_objects, frame_quad):
"""Bake selected projections into the active Image Empty."""
active = context.active_object
if active not in source_objects:
raise ValueError("The active Image Empty is not a Frame source")
view3d = context.region_data
sources = tuple(
_frame_source(
source,
context.region,
view3d,
view3d.view_matrix,
active=source is active,
)
for source in source_objects
)
active_source = next(source for source in sources if source["active"])
active_overlap = frame_source_overlap(active_source, frame_quad)
if not active_overlap:
raise ImageEditWarning("Frame does not overlap the active Image Empty")
output_size = frame_output_size(
frame_quad,
(context.region.width, context.region.height),
configured_max_frame_resolution(context),
)
pixels = composite_frame_pixels(sources, frame_quad, output_size)
depth_location = frame_depth_location(active_source, active_overlap)
matrix, display_size = frame_result_transform(
frame_quad,
depth_location,
context.region,
view3d,
output_size,
)
result_image = create_image_edit_result(active.data, pixels, output_size)
old_matrix = active.matrix_world.copy()
old_display_size = active.empty_display_size
old_offset = tuple(active.empty_image_offset)
source_images = []
for source in source_objects:
if all(image is not source.data for image in source_images):
source_images.append(source.data)
try:
active.matrix_world = matrix
active.empty_display_size = display_size
active.empty_image_offset = (-0.5, -0.5)
replace_empty_image(
active, result_image,
removed_objects=tuple(source for source in source_objects if source != active),
)
except Exception:
active.matrix_world = old_matrix
active.empty_display_size = old_display_size
active.empty_image_offset = old_offset
raise
for source in source_objects:
if source is not active:
bpy.data.objects.remove(source, do_unlink=True)
for image in source_images:
try:
if image.users == 0 and not image.use_fake_user:
bpy.data.images.remove(image, do_unlink=True)
except (ReferenceError, RuntimeError):
pass
return active
class FrameImages(bpy.types.Operator):
bl_idname = "anyimage.frame_images"
bl_label = "Frame Images"
bl_description = "Bake selected Image Empties into a view-facing frame"
bl_options = {"UNDO"}
_active_name = ""
_source_names = ()
_start = None
_cursor = None
_handle = None
_area_pointer = 0
_region_pointer = 0
@classmethod
def poll(cls, context):
return image_edit_poll(context)
def _draw_overlay(self):
if not drawing_in_region(self._area_pointer, self._region_pointer):
return
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
return
draw_polygon_fill(frame)
draw_dashed_line((*frame, frame[0]))
def _finish(self, context):
if self._handle is not None:
bpy.types.SpaceView3D.draw_handler_remove(self._handle, "WINDOW")
self._handle = None
context.workspace.status_text_set(None)
if context.area is not None:
context.area.tag_redraw()
def cancel(self, context):
self._finish(context)
def invoke(self, context, event):
try:
sources = selected_frame_sources(context)
except (TypeError, ValueError) as error:
report_image_edit_exception(self, error)
return {"CANCELLED"}
self._active_name = context.active_object.name
self._source_names = tuple(source.name for source in sources)
point = (float(event.mouse_region_x), float(event.mouse_region_y))
self._start = point
self._cursor = point
self._area_pointer = context.area.as_pointer()
self._region_pointer = context.region.as_pointer()
self._handle = bpy.types.SpaceView3D.draw_handler_add(
self._draw_overlay, (), "WINDOW", "POST_PIXEL"
)
context.workspace.status_text_set(
"Frame: drag a rectangle to merge selected Image Empties; "
"RMB or Esc cancels"
)
context.window_manager.modal_handler_add(self)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
def modal(self, context, event):
active_tool = active_view3d_tool_id(context)
if active_tool is not None and active_tool != FRAME_TOOL_ID:
self._finish(context)
return {"CANCELLED"}
if event.type in {"ESC", "RIGHTMOUSE"} and event.value == "PRESS":
self._finish(context)
return {"CANCELLED"}
if event.type == "MOUSEMOVE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
if event.type == "LEFTMOUSE" and event.value == "RELEASE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
self._finish(context)
return {"CANCELLED"}
try:
active = bpy.data.objects.get(self._active_name)
sources = tuple(
bpy.data.objects.get(name) for name in self._source_names
)
if (
not is_image_empty(active)
or any(not is_image_empty(source) for source in sources)
or context.active_object is not active
):
raise ValueError("The selected Image Empties changed during Frame")
apply_frame(context, sources, frame)
except (RuntimeError, TypeError, ValueError) as error:
report_image_edit_exception(self, error)
self._finish(context)
return {"CANCELLED"}
self._finish(context)
return {"FINISHED"}
return {"PASS_THROUGH"}
class FrameTool(bpy.types.WorkSpaceTool):
bl_space_type = "VIEW_3D"
bl_context_mode = "OBJECT"
bl_idname = FRAME_TOOL_ID
bl_label = "Frame"
bl_description = "Bake selected Image Empties into a view-facing frame"
bl_icon = "ops.sculpt.border_mask"
bl_operator = FrameImages.bl_idname
bl_keymap = image_edit_drag_keymap(FrameImages.bl_idname)

7
src/anyimage/operators/frame_tool/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
"""Frame conversion operators and workspace tool."""
from .operators import FrameImages, FrameTool
CLASSES = (FrameImages,)
__all__ = ("CLASSES", "FrameImages", "FrameTool")

141
src/anyimage/operators/frame_tool/compositing.py Normal file
View File

@@ -0,0 +1,141 @@
"""Composite projected image samples with independent RGB and Alpha coverage."""
import math
from ...common.image import (
PROJECTIVE_CHUNK_ROWS,
homography_from_points,
sample_rgba,
transform_homography,
)
from .projection import frame_source_coordinates
def _pixel_quad(image_size):
width, height = (float(value) for value in image_size)
if min(width, height) <= 0.0:
raise ValueError("The source image has no readable dimensions")
return ((0.0, 0.0), (width, 0.0), (width, height), (0.0, height))
def _composite_frame_samples(sources, screen_points):
"""Resolve independently sorted samples with an unweighted bottom RGB base."""
import numpy as np
samples, depths, coordinates = [], [], []
for source in sources:
points, depth, valid, _ = frame_source_coordinates(source, screen_points)
sample = sample_rgba(source["pixels"], points)
sample[~valid] = 0
depths.append(np.where(valid, depth, -math.inf))
samples.append(sample)
coordinates.append(points)
samples = np.stack(samples)
depths = np.stack(depths)
order = np.argsort(-depths, axis=0, kind="stable")
bottom = order[0]
covered = np.isfinite(np.take_along_axis(depths, bottom[None], axis=0)[0])
result = np.zeros_like(samples[0])
for index, source in enumerate(sources):
selected = covered & (bottom == index)
if np.any(selected):
result[..., :3][selected] = sample_rgba(
source["rgba"], coordinates[index][selected]
)[..., :3]
for index in range(len(sources)):
layer = np.take_along_axis(samples, order[index][None, ..., None], axis=0)[0]
alpha = layer[..., 3:4]
result[..., 3:4] = alpha + result[..., 3:4] * (1 - alpha)
if index:
result[..., :3] = layer[..., :3] + result[..., :3] * (1 - alpha)
return result, covered
def _frame_boundary_pixels(sources, points, output_to_screen):
"""Conservatively detect projected contour and depth-order crossings."""
import numpy as np
boundary = np.zeros(points.shape[:-1], dtype=bool)
for source in sources:
width, height = source["region_size"]
mapping = source["screen_to_source"] @ np.diag((1 / width, 1 / height, 1)) @ output_to_screen
u, v, w = mapping
lines = [u, v, u - w, v - w, w]
positive = np.ones_like(boundary)
negative = np.ones_like(boundary)
for line in (u, v, w - u, w - v, w):
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
positive &= value + radius >= 0
negative &= value - radius <= 0
source_boundary = np.zeros_like(boundary)
if source["perspective"]:
lines.append(source["source_to_view"][2] @ mapping)
for line in lines:
radius = 0.5 * (abs(line[0]) + abs(line[1]))
value = points[..., 0] * line[0] + points[..., 1] * line[1] + line[2]
source_boundary |= np.abs(value) < radius - 1e-10 * max(radius, 1e-20)
boundary |= source_boundary & (positive | negative)
if len(sources) > 1:
first_order = None
for offset in ((-0.499999, -0.499999), (0.499999, -0.499999),
(-0.499999, 0.499999), (0.499999, 0.499999)):
screen = transform_homography(points + offset, output_to_screen)
depths = []
for source in sources:
_, depth, valid, _ = frame_source_coordinates(source, screen)
depths.append(np.where(valid, depth, -math.inf))
order = np.argsort(-np.stack(depths), axis=0, kind="stable")
if first_order is None:
first_order = order
else:
boundary |= np.any(order != first_order, axis=0)
return boundary
def composite_frame_pixels(sources, frame_quad, output_size):
"""Composite projected Image Empty sources into one Blender-order RGBA image."""
import numpy as np
output_width, output_height = (int(value) for value in output_size)
if min(output_width, output_height) <= 0:
raise ValueError("The Frame output dimensions must be positive")
sources = tuple(
sorted(sources, key=lambda source: (source["active"], source["name"]))
)
if not sources or not sources[-1]["active"]:
raise ValueError("Frame requires one active Image Empty")
output_quad = _pixel_quad(output_size)
output_to_screen = homography_from_points(output_quad, frame_quad)
output = np.zeros((output_height, output_width, 4), dtype=np.float32)
covered = np.zeros((output_height, output_width), dtype=bool)
target_x = np.arange(output_width, dtype=np.float64) + 0.5
for row_start in range(0, output_height, PROJECTIVE_CHUNK_ROWS):
row_end = min(row_start + PROJECTIVE_CHUNK_ROWS, output_height)
target_y = np.arange(row_start, row_end, dtype=np.float64) + 0.5
x, y = np.meshgrid(target_x, target_y)
points = np.stack((x, y), axis=-1)
composite, coverage = _composite_frame_samples(
sources, transform_homography(points, output_to_screen)
)
boundary = _frame_boundary_pixels(sources, points, output_to_screen)
if np.any(boundary):
edge_points = points[boundary]
accumulated = np.zeros((len(edge_points), 4), dtype=np.float32)
counts = np.zeros(len(edge_points), dtype=np.int32)
for dy in (-0.375, -0.125, 0.125, 0.375):
for dx in (-0.375, -0.125, 0.125, 0.375):
sample, hit = _composite_frame_samples(
sources,
transform_homography(edge_points + (dx, dy), output_to_screen),
)
accumulated += sample
counts += hit
accumulated[..., 3] /= 16
accumulated[..., :3] /= np.maximum(counts, 1)[..., None]
composite[boundary] = accumulated
coverage[boundary] = counts > 0
covered[row_start:row_end] = coverage
output[row_start:row_end] = composite
output[..., 3][output[..., 3] <= 1e-8] = 0.0
output[~covered] = 0.0
return np.flipud(output).ravel()

264
src/anyimage/operators/frame_tool/operators.py Normal file
View File

@@ -0,0 +1,264 @@
"""Frame interaction, source preparation, and Blender image replacement."""
import bpy
from ...common.image import (
create_image_edit_result,
image_empty_bounds,
image_pixels,
is_image_empty,
premultiplied_rgba,
replace_empty_image,
is_animated_image,
)
from ...preferences import configured_max_frame_resolution
from ...common.viewport import (
active_image_edit_source,
active_view3d_tool_id,
draw_dashed_line,
draw_polygon_fill,
drawing_in_region,
image_edit_drag_keymap,
image_edit_poll,
rectangle_screen_path,
report_image_edit_exception,
)
from ...common.selection import ImageEditWarning
from .projection import (
frame_depth_location,
frame_output_size,
frame_result_transform,
frame_source_overlap,
frame_source_projection,
)
from .compositing import composite_frame_pixels
FRAME_TOOL_ID = "anyimage.frame"
def _frame_source(source_object, region, view3d, view_matrix, *, active):
import numpy as np
image_size = tuple(source_object.data.size)
pixels = image_pixels(source_object.data)
width, height = image_size
bounds = image_empty_bounds(source_object)
source = frame_source_projection(
view3d.perspective_matrix,
view_matrix,
source_object.matrix_world,
bounds,
(region.width, region.height),
perspective=view3d.is_perspective,
)
source.update(
{
"active": bool(active),
"image_size": image_size,
"name": source_object.name,
"object": source_object,
"pixels": premultiplied_rgba(pixels, image_size),
"rgba": np.flipud(pixels.reshape((height, width, 4))),
}
)
return source
def selected_frame_sources(context):
"""Return selected still Image Empties with the active object last."""
active = active_image_edit_source(context)
sources = [
source for source in context.selected_objects if is_image_empty(source)
]
for source in sources:
if is_animated_image(source.data):
raise ValueError("Frame only supports still images")
return tuple(
sorted(sources, key=lambda source: (source is active, source.name))
)
def apply_frame(context, source_objects, frame_quad):
"""Bake selected projections into the active Image Empty."""
active = context.active_object
if active not in source_objects:
raise ValueError("The active Image Empty is not a Frame source")
view3d = context.region_data
sources = tuple(
_frame_source(
source,
context.region,
view3d,
view3d.view_matrix,
active=source is active,
)
for source in source_objects
)
active_source = next(source for source in sources if source["active"])
active_overlap = frame_source_overlap(active_source, frame_quad)
if not active_overlap:
raise ImageEditWarning("Frame does not overlap the active Image Empty")
output_size = frame_output_size(
frame_quad,
(context.region.width, context.region.height),
configured_max_frame_resolution(context),
)
pixels = composite_frame_pixels(sources, frame_quad, output_size)
depth_location = frame_depth_location(active_source, active_overlap)
matrix, display_size = frame_result_transform(
frame_quad,
depth_location,
context.region,
view3d,
output_size,
)
result_image = create_image_edit_result(active.data, pixels, output_size)
old_matrix = active.matrix_world.copy()
old_display_size = active.empty_display_size
old_offset = tuple(active.empty_image_offset)
source_images = []
for source in source_objects:
if all(image is not source.data for image in source_images):
source_images.append(source.data)
try:
active.matrix_world = matrix
active.empty_display_size = display_size
active.empty_image_offset = (-0.5, -0.5)
replace_empty_image(
active, result_image,
removed_objects=tuple(source for source in source_objects if source != active),
)
except Exception:
active.matrix_world = old_matrix
active.empty_display_size = old_display_size
active.empty_image_offset = old_offset
raise
for source in source_objects:
if source is not active:
bpy.data.objects.remove(source, do_unlink=True)
for image in source_images:
try:
if image.users == 0 and not image.use_fake_user:
bpy.data.images.remove(image, do_unlink=True)
except (ReferenceError, RuntimeError):
pass
return active
class FrameImages(bpy.types.Operator):
bl_idname = "anyimage.frame_images"
bl_label = "Frame Images"
bl_description = "Draw a rectangle to combine selected images into one image from the current view."
bl_options = {"UNDO"}
_active_name = ""
_source_names = ()
_start = None
_cursor = None
_handle = None
_area_pointer = 0
_region_pointer = 0
@classmethod
def poll(cls, context):
return image_edit_poll(context)
def _draw_overlay(self):
if not drawing_in_region(self._area_pointer, self._region_pointer):
return
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
return
draw_polygon_fill(frame)
draw_dashed_line((*frame, frame[0]))
def _finish(self, context):
if self._handle is not None:
bpy.types.SpaceView3D.draw_handler_remove(self._handle, "WINDOW")
self._handle = None
context.workspace.status_text_set(None)
if context.area is not None:
context.area.tag_redraw()
def cancel(self, context):
self._finish(context)
def invoke(self, context, event):
try:
sources = selected_frame_sources(context)
except (TypeError, ValueError) as error:
report_image_edit_exception(self, error)
return {"CANCELLED"}
self._active_name = context.active_object.name
self._source_names = tuple(source.name for source in sources)
point = (float(event.mouse_region_x), float(event.mouse_region_y))
self._start = point
self._cursor = point
self._area_pointer = context.area.as_pointer()
self._region_pointer = context.region.as_pointer()
self._handle = bpy.types.SpaceView3D.draw_handler_add(
self._draw_overlay, (), "WINDOW", "POST_PIXEL"
)
context.workspace.status_text_set(
"Frame: drag a rectangle to merge selected Image Empties; "
"RMB or Esc cancels"
)
context.window_manager.modal_handler_add(self)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
def modal(self, context, event):
active_tool = active_view3d_tool_id(context)
if active_tool is not None and active_tool != FRAME_TOOL_ID:
self._finish(context)
return {"CANCELLED"}
if event.type in {"ESC", "RIGHTMOUSE"} and event.value == "PRESS":
self._finish(context)
return {"CANCELLED"}
if event.type == "MOUSEMOVE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
context.area.tag_redraw()
return {"RUNNING_MODAL"}
if event.type == "LEFTMOUSE" and event.value == "RELEASE":
self._cursor = (
float(event.mouse_region_x),
float(event.mouse_region_y),
)
frame = rectangle_screen_path(self._start, self._cursor)
if not frame:
self._finish(context)
return {"CANCELLED"}
try:
active = bpy.data.objects.get(self._active_name)
sources = tuple(
bpy.data.objects.get(name) for name in self._source_names
)
if (
not is_image_empty(active)
or any(not is_image_empty(source) for source in sources)
or context.active_object is not active
):
raise ValueError("The selected Image Empties changed during Frame")
apply_frame(context, sources, frame)
except (RuntimeError, TypeError, ValueError) as error:
report_image_edit_exception(self, error)
self._finish(context)
return {"CANCELLED"}
self._finish(context)
return {"FINISHED"}
return {"PASS_THROUGH"}
class FrameTool(bpy.types.WorkSpaceTool):
bl_space_type = "VIEW_3D"
bl_context_mode = "OBJECT"
bl_idname = FRAME_TOOL_ID
bl_label = "Frame"
bl_description = "Draw a rectangle to combine selected images into one image from the current view."
bl_icon = "ops.sculpt.border_mask"
bl_operator = FrameImages.bl_idname
bl_keymap = image_edit_drag_keymap(FrameImages.bl_idname)

264
src/anyimage/operators/frame_tool/projection.py Normal file
View File

@@ -0,0 +1,264 @@
"""Project image planes and place the view-facing Frame result."""
import math
from ...common.image import MAX_PROJECTIVE_OUTPUT_PIXELS
from ...common.selection import clip_polygon_halfplanes
PROJECTIVE_RELATIVE_EPSILON = 1e-12
def _matrix_array(matrix):
import numpy as np
return np.asarray(
tuple(tuple(float(value) for value in row) for row in matrix),
dtype=np.float64,
)
def frame_source_projection(
projection_matrix,
view_matrix,
matrix_world,
bounds,
region_size,
*,
perspective,
):
"""Return normalized image-plane transforms for one frozen view."""
import numpy as np
x_min, x_max, y_min, y_max = (float(value) for value in bounds)
region_width, region_height = (float(value) for value in region_size)
if min(x_max - x_min, y_max - y_min, region_width, region_height) <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_plane = np.asarray(
(
(x_max - x_min, 0.0, x_min),
(0.0, y_min - y_max, y_max),
(0.0, 0.0, 0.0),
(0.0, 0.0, 1.0),
),
dtype=np.float64,
)
world = _matrix_array(matrix_world)
source_to_world = world @ source_plane
source_to_view = _matrix_array(view_matrix) @ source_to_world
source_to_clip = _matrix_array(projection_matrix) @ source_to_world
source_to_screen = np.stack(
(
0.5 * (source_to_clip[0] + source_to_clip[3]),
0.5 * (source_to_clip[1] + source_to_clip[3]),
source_to_clip[3],
)
)
scale = float(np.max(np.abs(source_to_screen)))
if not math.isfinite(scale) or scale <= 0.0:
raise ValueError("The Image Empty projection is invalid")
source_to_screen /= scale
if not np.isfinite(source_to_screen).all():
raise ValueError("The Image Empty projection is invalid")
condition = float(np.linalg.cond(source_to_screen))
if not math.isfinite(condition) or condition > 1.0 / np.finfo(np.float64).eps:
raise ValueError("The Image Empty is edge-on to the current view")
return {
"bounds": (x_min, x_max, y_min, y_max),
"origin_depth": -float((_matrix_array(view_matrix) @ world)[2, 3]),
"perspective": bool(perspective),
"region_size": (region_width, region_height),
"screen_to_source": np.linalg.inv(source_to_screen),
"source_to_screen": source_to_screen,
"source_to_view": source_to_view,
"source_to_world": source_to_world,
}
def _projective_points(points, matrix):
"""Map points and return finite results plus per-point validity."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
flat = points.reshape((-1, 2))
homogeneous = np.column_stack((flat, np.ones(len(flat), dtype=np.float64)))
mapped = homogeneous @ np.asarray(matrix, dtype=np.float64).T
magnitude = np.max(np.abs(mapped), axis=1)
valid = (
np.isfinite(mapped).all(axis=1)
& (np.abs(mapped[:, 2]) > PROJECTIVE_RELATIVE_EPSILON * magnitude)
)
result = np.full((len(flat), 2), np.nan, dtype=np.float64)
result[valid] = mapped[valid, :2] / mapped[valid, 2, None]
return result.reshape(points.shape), valid.reshape(points.shape[:-1])
def frame_source_coordinates(source, screen_points):
"""Return source pixels, view depth, validity, and normalized coordinates."""
import numpy as np
screen_points = np.asarray(screen_points, dtype=np.float64)
region_width, region_height = source["region_size"]
normalized_screen = screen_points / (region_width, region_height)
normalized_source, valid = _projective_points(
normalized_screen,
source["screen_to_source"],
)
flat_source = normalized_source.reshape((-1, 2))
homogeneous = np.column_stack(
(flat_source, np.ones(len(flat_source), dtype=np.float64))
)
depth = -(homogeneous @ source["source_to_view"][2])
depth = depth.reshape(valid.shape)
valid &= np.isfinite(depth)
if source["perspective"]:
valid &= depth > 0.0
valid &= (
(normalized_source[..., 0] >= 0.0)
& (normalized_source[..., 0] <= 1.0)
& (normalized_source[..., 1] >= 0.0)
& (normalized_source[..., 1] <= 1.0)
)
image_width, image_height = source["image_size"]
source_points = normalized_source * (image_width, image_height)
source_points[~valid] = -1.0
return source_points, depth, valid, normalized_source
def frame_source_overlap(source, frame_quad):
"""Clip the source plane before dividing its homogeneous projection."""
import numpy as np
frame = np.asarray(frame_quad, dtype=np.float64) / source["region_size"]
left, bottom = frame.min(axis=0)
right, top = frame.max(axis=0)
x, y, w = source["source_to_screen"]
boundaries = [x - left * w, right * w - x, y - bottom * w, top * w - y]
if source["perspective"]:
boundaries.insert(0, -source["source_to_view"][2])
polygon = clip_polygon_halfplanes(
((0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0)), boundaries
)
if len(polygon) < 3:
return ()
screen, valid = _projective_points(polygon, source["source_to_screen"])
if not np.all(valid):
return ()
screen *= source["region_size"]
# Center the coordinates to avoid cancellation in a distant small polygon.
centered = screen - screen[0]
area = abs(np.sum(centered[:, 0] * np.roll(centered[:, 1], -1)
- centered[:, 1] * np.roll(centered[:, 0], -1))) * 0.5
return tuple(map(tuple, screen)) if area > 1e-8 else ()
def polygon_centroid(polygon):
"""Return the area centroid of one non-degenerate screen polygon."""
cross_sum = 0.0
x_sum = 0.0
y_sum = 0.0
for first, second in zip(polygon, (*polygon[1:], polygon[0])):
cross = first[0] * second[1] - second[0] * first[1]
cross_sum += cross
x_sum += (first[0] + second[0]) * cross
y_sum += (first[1] + second[1]) * cross
if abs(cross_sum) <= 1e-12:
raise ValueError("Frame does not overlap the active Image Empty")
return x_sum / (3.0 * cross_sum), y_sum / (3.0 * cross_sum)
def frame_output_size(frame_quad, region_size, max_resolution):
"""Choose output resolution from Frame pixels within the current Region."""
frame_width = abs(float(frame_quad[1][0] - frame_quad[0][0]))
frame_height = abs(float(frame_quad[0][1] - frame_quad[3][1]))
region_width, region_height = (float(value) for value in region_size)
maximum = float(max_resolution)
if min(frame_width, frame_height, region_width, region_height, maximum) <= 0.0:
raise ValueError("The Frame output dimensions must be positive")
scale = min(
1.0,
region_width / frame_width,
region_height / frame_height,
maximum / max(frame_width, frame_height),
math.sqrt(
MAX_PROJECTIVE_OUTPUT_PIXELS / max(frame_width * frame_height, 1.0)
),
)
return (
max(2, int(round(frame_width * scale))),
max(2, int(round(frame_height * scale))),
)
def view_facing_frame_geometry(corners, output_size):
"""Return a world matrix array and display size for frame-plane corners."""
import numpy as np
corners = np.asarray(corners, dtype=np.float64)
if corners.shape != (4, 3) or not np.isfinite(corners).all():
raise ValueError("The Frame view plane is invalid")
top_left, top_right, _bottom_right, bottom_left = corners
right = top_right - top_left
up = top_left - bottom_left
width = float(np.linalg.norm(right))
height = float(np.linalg.norm(up))
if min(width, height) <= 1e-10:
raise ValueError("Draw a larger Frame")
right /= width
up /= height
forward = np.cross(right, up)
forward_length = float(np.linalg.norm(forward))
if forward_length <= 1e-10:
raise ValueError("The Frame view plane is invalid")
forward /= forward_length
matrix = np.identity(4, dtype=np.float64)
matrix[:3, 0] = right
matrix[:3, 1] = up
matrix[:3, 2] = forward
matrix[:3, 3] = corners.mean(axis=0)
display_size = width if output_size[0] >= output_size[1] else height
return matrix, display_size
def frame_result_transform(frame_quad, source_origin, region, view3d, output_size):
"""Return the view-facing object matrix and Image Empty display size."""
from mathutils import Matrix
from bpy_extras import view3d_utils
corners = tuple(
tuple(
float(value)
for value in view3d_utils.region_2d_to_location_3d(
region,
view3d,
point,
source_origin,
)
)
for point in frame_quad
)
matrix, display_size = view_facing_frame_geometry(corners, output_size)
return Matrix(tuple(tuple(row) for row in matrix)), display_size
def frame_depth_location(active_source, overlap_polygon):
"""Return the world point whose view depth places the Frame result."""
import numpy as np
source_object = active_source["object"]
if not active_source["perspective"] or active_source["origin_depth"] > 0.0:
return source_object.matrix_world.translation.copy()
centroid = np.asarray((polygon_centroid(overlap_polygon),), dtype=np.float64)
_points, depth, valid, normalized_source = frame_source_coordinates(
active_source,
centroid,
)
if not bool(valid[0]) or depth[0] <= 0.0:
raise ValueError("The Frame result depth is invalid")
source_point = np.asarray(
(normalized_source[0, 0], normalized_source[0, 1], 1.0),
dtype=np.float64,
)
world = active_source["source_to_world"] @ source_point
if not np.isfinite(world).all() or abs(world[3]) <= 1e-12:
raise ValueError("The Frame result depth is invalid")
return tuple(float(value) for value in world[:3] / world[3])

6
src/anyimage/operators/mask_tool.py
View File

@@ -39,7 +39,7 @@ def compose_alpha(source_alpha, mask_values, mode):
mode = str(mode)
if mode == "SET":
result = alpha * mask
elif mode == "ADD":
elif mode == "EXTEND":
result = np.maximum(alpha, mask)
elif mode == "SUBTRACT":
result = alpha * (1.0 - mask)
@@ -72,7 +72,7 @@ def rasterize_brush_path(image_size, path, radius, project_path):
class EditImageAlpha(ImageGesture, bpy.types.Operator):
bl_idname = "anyimage.edit_image_alpha"
bl_label = "Edit Image Alpha"
bl_description = "Edit Image Empty Alpha with a lasso, brush, or polyline"
bl_description = "Hide or reveal image areas with a lasso, brush, or polyline."
bl_options = {"UNDO"}
active_tool_id = MASK_TOOL_ID
resolve_on_press = True
@@ -157,7 +157,7 @@ class MaskTool(bpy.types.WorkSpaceTool):
bl_context_mode = "OBJECT"
bl_idname = MASK_TOOL_ID
bl_label = "Mask"
bl_description = "Edit Image Empty Alpha with a lasso, brush, or polyline"
bl_description = "Hide or reveal image areas with a lasso, brush, or polyline."
bl_icon = "ops.sculpt.lasso_mask"
bl_operator = EditImageAlpha.bl_idname
bl_cursor = "PAINT_CROSS"

7
src/anyimage/operators/rectify_tool/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
"""Perspective correction operators and workspace tool."""
from .operators import RectifyImagePerspective, RectifyTool
CLASSES = (RectifyImagePerspective,)
__all__ = ("CLASSES", "RectifyImagePerspective", "RectifyTool")

144
src/anyimage/operators/rectify_tool/geometry.py Normal file
View File

@@ -0,0 +1,144 @@
"""Validate perspective quads and resample the selected image region."""
import math
from ...common.selection import (
clip_polygon_halfplanes,
trim_pixels_to_alpha,
)
from ...common.image import (
projective_output_size,
warp_projective_pixels,
)
MIN_INTERACTIVE_ASPECT = 0.05
MAX_INTERACTIVE_ASPECT = 20.0
def validate_perspective_quad(points, min_area=1e-6):
"""Return four convex corners without changing their correspondence."""
import numpy as np
quad = np.asarray(points, dtype=np.float64)
if quad.shape != (4, 2) or not np.isfinite(quad).all():
raise ValueError("Rectify requires four valid image corners")
edges = np.roll(quad, -1, axis=0) - quad
if np.any(np.linalg.norm(edges, axis=1) < 2.0):
raise ValueError("Rectify corners are too close together")
crosses = np.cross(edges, np.roll(edges, -1, axis=0))
if np.any(np.abs(crosses) <= 1e-8) or not (
np.all(crosses > 0.0) or np.all(crosses < 0.0)
):
raise ValueError("Rectify corners must define one convex quadrilateral")
area = 0.5 * abs(
float(
np.dot(quad[:, 0], np.roll(quad[:, 1], -1))
- np.dot(quad[:, 1], np.roll(quad[:, 0], -1))
)
)
if area < min_area:
raise ValueError("Draw a larger Rectify quad")
return quad
def canonical_perspective_quad(points, min_area=1e-6):
"""Order four unordered points as source-oriented top-left through bottom-left."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
if points.shape != (4, 2) or not np.isfinite(points).all():
raise ValueError("Rectify requires four valid image corners")
distances = np.linalg.norm(points[:, None] - points[None, :], axis=2)
distances += np.eye(4) * 1e9
if float(distances.min()) < 2.0:
raise ValueError("Rectify corners are too close together")
center = points.mean(axis=0)
ring = points[np.argsort(np.arctan2(points[:, 1] - center[1], points[:, 0] - center[0]))]
ring = validate_perspective_quad(ring, min_area)
candidates = []
for winding in (ring, ring[::-1]):
for start in range(4):
candidate = np.roll(winding, -start, axis=0)
top = candidate[1] - candidate[0]
left = candidate[3] - candidate[0]
score = top[0] / np.linalg.norm(top) + left[1] / np.linalg.norm(left)
candidates.append((float(score), -candidate[0, 1], -candidate[0, 0], candidate))
return max(candidates, key=lambda item: item[:3])[3]
def perspective_quad_overlaps_image(points, image_size):
import numpy as np
width, height = image_size
if min(width, height) <= 0:
raise ValueError("The source image has no readable dimensions")
clipped = np.asarray(clip_polygon_halfplanes(
points, ((1, 0, 0), (-1, 0, width), (0, 1, 0), (0, -1, height))
))
if clipped.ndim != 2 or len(clipped) < 3:
return False
area = 0.5 * abs(
float(
np.dot(clipped[:, 0], np.roll(clipped[:, 1], -1))
- np.dot(clipped[:, 1], np.roll(clipped[:, 0], -1))
)
)
return area > 1e-6
def perspective_quad_aspect(quad):
"""Estimate an initial output aspect from the visible quad edge lengths."""
import numpy as np
quad = validate_perspective_quad(quad)
width = 0.5 * (
np.linalg.norm(quad[1] - quad[0]) + np.linalg.norm(quad[2] - quad[3])
)
height = 0.5 * (
np.linalg.norm(quad[2] - quad[1]) + np.linalg.norm(quad[3] - quad[0])
)
return min(
max(float(width / max(height, 1e-8)), MIN_INTERACTIVE_ASPECT),
MAX_INTERACTIVE_ASPECT,
)
def perspective_placement_bounds(quad, output_size, trim_bounds):
"""Map a rectified Alpha trim back to an approximate source-image rectangle."""
quad = validate_perspective_quad(quad)
source_left = math.floor(float(quad[:, 0].min()))
source_top = math.floor(float(quad[:, 1].min()))
source_right = math.ceil(float(quad[:, 0].max()))
source_bottom = math.ceil(float(quad[:, 1].max()))
output_width, output_height = (float(value) for value in output_size)
trim_left, trim_top, trim_right, trim_bottom = trim_bounds
source_width = source_right - source_left
source_height = source_bottom - source_top
return (
source_left + trim_left * source_width / output_width,
source_top + trim_top * source_height / output_height,
source_left + trim_right * source_width / output_width,
source_top + trim_bottom * source_height / output_height,
)
def extract_perspective_pixels(
source_pixels,
source_size,
quad,
aspect_ratio,
):
output_size = projective_output_size(quad, aspect_ratio)
pixels = warp_projective_pixels(
source_pixels, source_size, quad, output_size
)
pixels, trim_bounds = trim_pixels_to_alpha(pixels, (0, 0, *output_size))
trimmed_size = (
trim_bounds[2] - trim_bounds[0],
trim_bounds[3] - trim_bounds[1],
)
placement_bounds = perspective_placement_bounds(
quad, output_size, trim_bounds
)
return pixels, trimmed_size, placement_bounds

326
src/anyimage/operators/rectify_tool.py → src/anyimage/operators/rectify_tool/operators.py
View File

@@ -1,9 +1,11 @@
"""Rectify interaction and Blender image replacement."""
import json
import math
import bpy
from ..common.viewport import (
from ...common.viewport import (
active_view3d_tool_id,
deserialize_matrix,
draw_dashed_line,
@@ -16,311 +18,45 @@ from ..common.viewport import (
screen_path_to_image_pixels,
serialize_matrix,
)
from ..common.selection import clip_polygon_halfplanes, trim_pixels_to_alpha
from ..common.image import (
from ...common.image import (
create_image_edit_result,
image_base_name,
image_pixels,
image_empty_bounds,
projective_output_size,
replace_empty_image,
require_image_empty,
video_total_frames,
is_animated_image,
warp_projective_pixels,
)
MIN_QUAD_AREA = 16.0
RECTIFY_TOOL_ID = "anyimage.rectify"
PREVIEW_TEXTURE_SIZE = 512
ASPECT_PIXELS_PER_DOUBLE = 240.0
MIN_INTERACTIVE_ASPECT = 0.05
MAX_INTERACTIVE_ASPECT = 20.0
ASPECT_PRESETS = (
("9 : 16", 9.0 / 16.0),
("3 : 4", 3.0 / 4.0),
("1 : 1", 1.0),
("4 : 3", 4.0 / 3.0),
("16 : 9", 16.0 / 9.0),
from .geometry import (
MAX_INTERACTIVE_ASPECT,
MIN_INTERACTIVE_ASPECT,
canonical_perspective_quad,
extract_perspective_pixels,
perspective_quad_aspect,
perspective_quad_overlaps_image,
validate_perspective_quad,
)
from .preview import (
PREVIEW_TEXTURE_SIZE,
aspect_ratio_from_mouse,
create_perspective_preview_image,
draw_aspect_hud,
draw_preview_frame,
interactive_preview_polygon,
preview_draw_bounds,
preview_texture_draw_options,
snapped_aspect_ratio,
)
def validate_perspective_quad(points, min_area=1e-6):
"""Return four convex corners without changing their correspondence."""
import numpy as np
quad = np.asarray(points, dtype=np.float64)
if quad.shape != (4, 2) or not np.isfinite(quad).all():
raise ValueError("Rectify requires four valid image corners")
edges = np.roll(quad, -1, axis=0) - quad
if np.any(np.linalg.norm(edges, axis=1) < 2.0):
raise ValueError("Rectify corners are too close together")
crosses = np.cross(edges, np.roll(edges, -1, axis=0))
if np.any(np.abs(crosses) <= 1e-8) or not (
np.all(crosses > 0.0) or np.all(crosses < 0.0)
):
raise ValueError("Rectify corners must define one convex quadrilateral")
area = 0.5 * abs(
float(
np.dot(quad[:, 0], np.roll(quad[:, 1], -1))
- np.dot(quad[:, 1], np.roll(quad[:, 0], -1))
)
)
if area < min_area:
raise ValueError("Draw a larger Rectify quad")
return quad
def canonical_perspective_quad(points, min_area=1e-6):
"""Order four unordered points as source-oriented top-left through bottom-left."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
if points.shape != (4, 2) or not np.isfinite(points).all():
raise ValueError("Rectify requires four valid image corners")
distances = np.linalg.norm(points[:, None] - points[None, :], axis=2)
distances += np.eye(4) * 1e9
if float(distances.min()) < 2.0:
raise ValueError("Rectify corners are too close together")
center = points.mean(axis=0)
ring = points[np.argsort(np.arctan2(points[:, 1] - center[1], points[:, 0] - center[0]))]
ring = validate_perspective_quad(ring, min_area)
candidates = []
for winding in (ring, ring[::-1]):
for start in range(4):
candidate = np.roll(winding, -start, axis=0)
top = candidate[1] - candidate[0]
left = candidate[3] - candidate[0]
score = top[0] / np.linalg.norm(top) + left[1] / np.linalg.norm(left)
candidates.append((float(score), -candidate[0, 1], -candidate[0, 0], candidate))
return max(candidates, key=lambda item: item[:3])[3]
def preview_polygon(points):
"""Return a non-crossing preview hull for the current unordered clicks."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
if len(points) < 3:
return points
center = points.mean(axis=0)
return points[np.argsort(np.arctan2(points[:, 1] - center[1], points[:, 0] - center[0]))]
def interactive_preview_polygon(points, cursor):
"""Return the clicked points plus one distinct moving cursor point."""
vertices = list(points)
if (
cursor is not None
and vertices
and len(vertices) < 4
and math.dist(vertices[-1], cursor) >= 2.0
):
vertices.append(cursor)
return preview_polygon(vertices)
def perspective_quad_overlaps_image(points, image_size):
import numpy as np
width, height = image_size
if min(width, height) <= 0:
raise ValueError("The source image has no readable dimensions")
clipped = np.asarray(clip_polygon_halfplanes(
points, ((1, 0, 0), (-1, 0, width), (0, 1, 0), (0, -1, height))
))
if clipped.ndim != 2 or len(clipped) < 3:
return False
area = 0.5 * abs(
float(
np.dot(clipped[:, 0], np.roll(clipped[:, 1], -1))
- np.dot(clipped[:, 1], np.roll(clipped[:, 0], -1))
)
)
return area > 1e-6
def aspect_ratio_from_mouse(initial_ratio, horizontal_delta):
ratio = initial_ratio * math.pow(
2.0, float(horizontal_delta) / ASPECT_PIXELS_PER_DOUBLE
)
return min(max(ratio, MIN_INTERACTIVE_ASPECT), MAX_INTERACTIVE_ASPECT)
def snapped_aspect_ratio(aspect_ratio):
label, ratio = min(
ASPECT_PRESETS,
key=lambda preset: abs(math.log(aspect_ratio / preset[1])),
)
return ratio, label
def aspect_ratio_label(aspect_ratio, preset_label=None):
if preset_label is not None:
return preset_label
if aspect_ratio >= 1.0:
return f"{aspect_ratio:.2f} : 1"
return f"1 : {1.0 / aspect_ratio:.2f}"
def preview_draw_bounds(region_size, aspect_ratio):
region_width, region_height = (float(value) for value in region_size)
available_width = min(region_width * 0.65, 640.0)
available_height = min(region_height * 0.65, 640.0)
if aspect_ratio >= available_width / available_height:
width = available_width
height = width / aspect_ratio
else:
height = available_height
width = height * aspect_ratio
return (
(region_width - width) * 0.5,
(region_height - height) * 0.5,
width,
height,
)
def draw_preview_frame(bounds):
import gpu
from gpu_extras.batch import batch_for_shader
left, bottom, width, height = bounds
vertices = (
(left, bottom),
(left + width, bottom),
(left + width, bottom + height),
(left, bottom + height),
(left, bottom),
)
shader = gpu.shader.from_builtin("UNIFORM_COLOR")
batch = batch_for_shader(shader, "LINE_STRIP", {"pos": vertices})
gpu.state.blend_set("ALPHA")
gpu.state.line_width_set(2.0)
shader.bind()
shader.uniform_float("color", (0.12, 0.52, 1.0, 1.0))
batch.draw(shader)
gpu.state.line_width_set(1.0)
gpu.state.blend_set("NONE")
def draw_centered_text(text, center_x, baseline_y, size, color):
import blf
font_id = 0
blf.size(font_id, size)
width, _height = blf.dimensions(font_id, text)
x = center_x - width * 0.5
blf.position(font_id, x + 1.0, baseline_y - 1.0, 0.0)
blf.color(font_id, 0.0, 0.0, 0.0, 0.9)
blf.draw(font_id, text)
blf.position(font_id, x, baseline_y, 0.0)
blf.color(font_id, *color)
blf.draw(font_id, text)
def draw_aspect_hud(bounds, aspect_ratio, preset_label=None):
left, bottom, width, height = bounds
center_x = left + width * 0.5
draw_centered_text(
aspect_ratio_label(aspect_ratio, preset_label),
center_x,
bottom + height * 0.5 - 10.0,
22,
(0.25, 0.65, 1.0, 1.0),
)
draw_centered_text(
"Move horizontally • Ctrl snap • LMB confirm • Backspace edit",
center_x,
bottom - 26.0,
13,
(0.85, 0.9, 1.0, 1.0),
)
def create_perspective_preview_image(source_image, pixels):
image = bpy.data.images.new(
f"{image_base_name(source_image)}.rectify-preview",
width=PREVIEW_TEXTURE_SIZE,
height=PREVIEW_TEXTURE_SIZE,
alpha=True,
)
try:
image.colorspace_settings.name = "sRGB"
image.alpha_mode = "STRAIGHT"
image.pixels.foreach_set(pixels)
image.update()
except Exception:
bpy.data.images.remove(image, do_unlink=True)
raise
return image
def preview_texture_draw_options(blender_version):
if tuple(blender_version) >= (5, 0, 0):
return {"is_scene_linear_with_rec709_srgb_target": True}
return {}
def perspective_quad_aspect(quad):
"""Estimate an initial output aspect from the visible quad edge lengths."""
import numpy as np
quad = validate_perspective_quad(quad)
width = 0.5 * (
np.linalg.norm(quad[1] - quad[0]) + np.linalg.norm(quad[2] - quad[3])
)
height = 0.5 * (
np.linalg.norm(quad[2] - quad[1]) + np.linalg.norm(quad[3] - quad[0])
)
return min(
max(float(width / max(height, 1e-8)), MIN_INTERACTIVE_ASPECT),
MAX_INTERACTIVE_ASPECT,
)
def perspective_placement_bounds(quad, output_size, trim_bounds):
"""Map a rectified Alpha trim back to an approximate source-image rectangle."""
quad = validate_perspective_quad(quad)
source_left = math.floor(float(quad[:, 0].min()))
source_top = math.floor(float(quad[:, 1].min()))
source_right = math.ceil(float(quad[:, 0].max()))
source_bottom = math.ceil(float(quad[:, 1].max()))
output_width, output_height = (float(value) for value in output_size)
trim_left, trim_top, trim_right, trim_bottom = trim_bounds
source_width = source_right - source_left
source_height = source_bottom - source_top
return (
source_left + trim_left * source_width / output_width,
source_top + trim_top * source_height / output_height,
source_left + trim_right * source_width / output_width,
source_top + trim_bottom * source_height / output_height,
)
def extract_perspective_pixels(
source_pixels,
source_size,
quad,
aspect_ratio,
):
output_size = projective_output_size(quad, aspect_ratio)
pixels = warp_projective_pixels(
source_pixels, source_size, quad, output_size
)
pixels, trim_bounds = trim_pixels_to_alpha(pixels, (0, 0, *output_size))
trimmed_size = (
trim_bounds[2] - trim_bounds[0],
trim_bounds[3] - trim_bounds[1],
)
placement_bounds = perspective_placement_bounds(
quad, output_size, trim_bounds
)
return pixels, trimmed_size, placement_bounds
MIN_QUAD_AREA = 16.0
RECTIFY_TOOL_ID = "anyimage.rectify"
class RectifyImagePerspective(bpy.types.Operator):
bl_idname = "anyimage.rectify_image_perspective"
bl_label = "Rectify Image Perspective"
bl_description = "Rectify four image corners into a rectangular image"
bl_description = "Select four corners of an image area to correct its perspective into a rectangle."
bl_options = {"REGISTER", "UNDO"}
source_object_name: bpy.props.StringProperty(options={"HIDDEN", "SKIP_SAVE"})
@@ -456,7 +192,7 @@ class RectifyImagePerspective(bpy.types.Operator):
return {"FINISHED"} if source_object is not None else {"CANCELLED"}
if source_object is None:
return {"CANCELLED"}
if video_total_frames(source_object.data) > 1:
if is_animated_image(source_object.data):
self.report(
{"WARNING"},
"Rectify only supports a still image",
@@ -621,7 +357,7 @@ class RectifyImagePerspective(bpy.types.Operator):
self.report({"ERROR"}, str(error))
return {"CANCELLED"}
try:
if video_total_frames(source_object.data) > 1:
if is_animated_image(source_object.data):
raise RuntimeError("Rectify only supports a still image")
quad = validate_perspective_quad(json.loads(self.quad_json))
if not perspective_quad_overlaps_image(
@@ -632,8 +368,6 @@ class RectifyImagePerspective(bpy.types.Operator):
"Rectify quad does not overlap the source image",
)
return {"CANCELLED"}
import numpy as np
source_pixels = self._source_pixels
if source_pixels is None or len(source_pixels) != len(
source_object.data.pixels
@@ -672,7 +406,7 @@ class RectifyTool(bpy.types.WorkSpaceTool):
bl_context_mode = "OBJECT"
bl_idname = RECTIFY_TOOL_ID
bl_label = "Rectify"
bl_description = "Rectify four image corners into a rectangular image"
bl_description = "Select four corners of an image area to correct its perspective into a rectangle."
bl_icon = "ops.sculpt.line_project"
bl_operator = RectifyImagePerspective.bl_idname
bl_keymap = image_edit_point_keymap(RectifyImagePerspective.bl_idname)

168
src/anyimage/operators/rectify_tool/preview.py Normal file
View File

@@ -0,0 +1,168 @@
"""Rectify preview images, overlays, and interactive aspect controls."""
import math
import bpy
from ...common.image import image_base_name
from .geometry import (
MAX_INTERACTIVE_ASPECT,
MIN_INTERACTIVE_ASPECT,
)
PREVIEW_TEXTURE_SIZE = 512
ASPECT_PIXELS_PER_DOUBLE = 240.0
ASPECT_PRESETS = (
("9 : 16", 9.0 / 16.0),
("3 : 4", 3.0 / 4.0),
("1 : 1", 1.0),
("4 : 3", 4.0 / 3.0),
("16 : 9", 16.0 / 9.0),
)
def preview_polygon(points):
"""Return a non-crossing preview hull for the current unordered clicks."""
import numpy as np
points = np.asarray(points, dtype=np.float64)
if len(points) < 3:
return points
center = points.mean(axis=0)
return points[np.argsort(np.arctan2(points[:, 1] - center[1], points[:, 0] - center[0]))]
def interactive_preview_polygon(points, cursor):
"""Return the clicked points plus one distinct moving cursor point."""
vertices = list(points)
if (
cursor is not None
and vertices
and len(vertices) < 4
and math.dist(vertices[-1], cursor) >= 2.0
):
vertices.append(cursor)
return preview_polygon(vertices)
def aspect_ratio_from_mouse(initial_ratio, horizontal_delta):
ratio = initial_ratio * math.pow(
2.0, float(horizontal_delta) / ASPECT_PIXELS_PER_DOUBLE
)
return min(max(ratio, MIN_INTERACTIVE_ASPECT), MAX_INTERACTIVE_ASPECT)
def snapped_aspect_ratio(aspect_ratio):
label, ratio = min(
ASPECT_PRESETS,
key=lambda preset: abs(math.log(aspect_ratio / preset[1])),
)
return ratio, label
def aspect_ratio_label(aspect_ratio, preset_label=None):
if preset_label is not None:
return preset_label
if aspect_ratio >= 1.0:
return f"{aspect_ratio:.2f} : 1"
return f"1 : {1.0 / aspect_ratio:.2f}"
def preview_draw_bounds(region_size, aspect_ratio):
region_width, region_height = (float(value) for value in region_size)
available_width = min(region_width * 0.65, 640.0)
available_height = min(region_height * 0.65, 640.0)
if aspect_ratio >= available_width / available_height:
width = available_width
height = width / aspect_ratio
else:
height = available_height
width = height * aspect_ratio
return (
(region_width - width) * 0.5,
(region_height - height) * 0.5,
width,
height,
)
def draw_preview_frame(bounds):
import gpu
from gpu_extras.batch import batch_for_shader
left, bottom, width, height = bounds
vertices = (
(left, bottom),
(left + width, bottom),
(left + width, bottom + height),
(left, bottom + height),
(left, bottom),
)
shader = gpu.shader.from_builtin("UNIFORM_COLOR")
batch = batch_for_shader(shader, "LINE_STRIP", {"pos": vertices})
gpu.state.blend_set("ALPHA")
gpu.state.line_width_set(2.0)
shader.bind()
shader.uniform_float("color", (0.12, 0.52, 1.0, 1.0))
batch.draw(shader)
gpu.state.line_width_set(1.0)
gpu.state.blend_set("NONE")
def draw_centered_text(text, center_x, baseline_y, size, color):
import blf
font_id = 0
blf.size(font_id, size)
width, _height = blf.dimensions(font_id, text)
x = center_x - width * 0.5
blf.position(font_id, x + 1.0, baseline_y - 1.0, 0.0)
blf.color(font_id, 0.0, 0.0, 0.0, 0.9)
blf.draw(font_id, text)
blf.position(font_id, x, baseline_y, 0.0)
blf.color(font_id, *color)
blf.draw(font_id, text)
def draw_aspect_hud(bounds, aspect_ratio, preset_label=None):
left, bottom, width, height = bounds
center_x = left + width * 0.5
draw_centered_text(
aspect_ratio_label(aspect_ratio, preset_label),
center_x,
bottom + height * 0.5 - 10.0,
22,
(0.25, 0.65, 1.0, 1.0),
)
draw_centered_text(
"Move horizontally • Ctrl snap • LMB confirm • Backspace edit",
center_x,
bottom - 26.0,
13,
(0.85, 0.9, 1.0, 1.0),
)
def create_perspective_preview_image(source_image, pixels):
image = bpy.data.images.new(
f"{image_base_name(source_image)}.rectify-preview",
width=PREVIEW_TEXTURE_SIZE,
height=PREVIEW_TEXTURE_SIZE,
alpha=True,
)
try:
image.colorspace_settings.name = "sRGB"
image.alpha_mode = "STRAIGHT"
image.pixels.foreach_set(pixels)
image.update()
except Exception:
bpy.data.images.remove(image, do_unlink=True)
raise
return image
def preview_texture_draw_options(blender_version):
if tuple(blender_version) >= (5, 0, 0):
return {"is_scene_linear_with_rec709_srgb_target": True}
return {}

7
src/anyimage/operators/remove_background.py
View File

@@ -86,23 +86,20 @@ class RunBackgroundRemoval(JobOperatorBase, bpy.types.Operator):
require_environment()
model = configured_background_model(context)
require_model(model)
settings = context.scene.anyimage_settings
parameters = {
"input": str(require_input_path(self.input_path)),
"model": model,
"device": production_device().lower(),
"video_frames": settings.video_frames,
"start_frame": self._image_target.start_frame,
}
if getattr(self, "_hdr_input", None) is not None:
parameters.update(output_kind="alpha", video_frames=1, start_frame=1)
parameters["output_kind"] = "alpha"
return parameters
def response(self, _context, result):
if getattr(self, "_hdr_input", None) is not None:
self._hdr_input.apply(self._image_target, result.file("alpha"), prepare_undo=True)
else:
self._image_target.apply(result.directory)
self._image_target.apply(result.file("foreground"))
if "FINISHED" not in bpy.ops.ed.undo_push(message="Remove Background"):
raise RuntimeError("Unable to create the Remove Background Undo step")
return "Image background is removed"

5
src/anyimage/operators/upscale.py
View File

@@ -115,18 +115,15 @@ class UpscaleImage(JobOperatorBase, bpy.types.Operator):
def request(self, context):
require_environment()
require_model(self.model)
settings = context.scene.anyimage_settings
return {
"input": str(require_input_path(self.input_path)),
"model": self.model,
"device": production_device().lower(),
"video_frames": settings.video_frames,
"start_frame": self._image_target.start_frame,
"max_input_size": int(self.max_input_size),
}
def response(self, _context, result):
self._image_target.apply(result.directory, frame_prefix="upscale.")
self._image_target.apply(result.file("upscale"))
model_label = upscale_model_label(self.model)
return f"Image is upscaled with {model_label}"

6
src/anyimage/panel.py
View File

@@ -87,15 +87,15 @@ class ServerPanel(bpy.types.Panel):
cached_row = layout.row(align=True)
if len(cached_names) > 1:
cached_list = cached_row.column(align=True)
cached_list.label(text="Cached:")
cached_list.label(text="Loaded Models:")
for name in cached_names:
cached_list.label(text=name)
else:
cached_row.label(
text=(
f"Cached: {cached_names[0]}"
f"Loaded Models: {cached_names[0]}"
if cached_names
else "Cached: None"
else "No Models Loaded"
)
)
clear = cached_row.row(align=True)

42
src/anyimage/preferences.py
View File

@@ -39,9 +39,9 @@ BALLOON_PROFILE_ALGORITHM_ITEMS = (
(
"POISSON",
"Poisson",
"Smooth cotangent-Laplacian Balloon profile",
"Create a smooth, rounded bulge within the Cutout outline.",
),
("TEDDY", "Teddy", "Chordal-axis Balloon profile"),
("TEDDY", "Teddy", "Create a bulge shaped by the Cutout's central branches."),
)
@@ -136,27 +136,25 @@ class AnyImagePreferences(bpy.types.AddonPreferences):
bl_idname = ADDON_PACKAGE
adapt_material_to_view_transform: bpy.props.BoolProperty(
name="Adapt to Scene View Transform",
name="Preserve Image Colors",
description=(
"Use the scene view transform's inverse color space with Straight alpha "
"for new material color textures; otherwise use sRGB with Premultiplied alpha"
"Compensate for the scene's view transform to keep image colors "
"close to the original when creating materials."
),
default=True,
)
storage_root: bpy.props.StringProperty(
name="Storage Root",
name="Storage Folder",
description=(
"Shared user-writable storage for the environment, models and jobs; "
"all Blender versions can use the same directory"
"Choose where AnyImage stores its AI environment, downloaded models, and task files."
),
subtype="DIR_PATH",
default=str(DEFAULT_STORAGE_ROOT),
)
device: bpy.props.EnumProperty(
name="Device",
name="AI Processing Device",
description=(
"Inference device used by production tasks; only devices supported "
"on this machine are listed"
"Choose the CPU or GPU used to run AI models."
),
items=device_items,
default=0,
@@ -170,8 +168,8 @@ class AnyImagePreferences(bpy.types.AddonPreferences):
max_ai_input_size: bpy.props.IntProperty(
name="Maximum AI Input Size",
description=(
"Maximum source long edge eligible for Upscale and maximum long edge "
"used by Depth, Normal and Selection refinement"
"Limit the longest image edge used for depth, normal, and selection refinement. "
"Upscale requires a source image smaller than this limit."
),
default=DEFAULT_MAX_AI_INPUT_SIZE,
min=MIN_MAX_AI_INPUT_SIZE,
@@ -181,7 +179,10 @@ class AnyImagePreferences(bpy.types.AddonPreferences):
)
max_cutout_mesh_resolution: bpy.props.IntProperty(
name="Maximum Cutout Mesh Resolution",
description="Maximum sampling resolution along the longest Cutout edge",
description=(
"Limit the longest image edge used to sample the Cutout mesh. "
"Higher values allow finer outlines and denser meshes."
),
default=DEFAULT_MAX_CUTOUT_MESH_RESOLUTION,
min=MIN_CUTOUT_MESH_RESOLUTION,
max=MAX_CUTOUT_MESH_RESOLUTION,
@@ -190,7 +191,7 @@ class AnyImagePreferences(bpy.types.AddonPreferences):
)
max_frame_resolution: bpy.props.IntProperty(
name="Maximum Frame Resolution",
description="Maximum output resolution along the longest Frame edge",
description="Set the maximum length, in pixels, of the combined image's longest edge.",
default=DEFAULT_MAX_FRAME_RESOLUTION,
min=MIN_FRAME_RESOLUTION,
max=MAX_FRAME_RESOLUTION,
@@ -199,7 +200,7 @@ class AnyImagePreferences(bpy.types.AddonPreferences):
)
balloon_profile_algorithm: bpy.props.EnumProperty(
name="Balloon Profile",
description="Algorithm used to generate o_balloon for every Cutout shape",
description="Control how the Cutout surface bulges outward from its outline.",
items=BALLOON_PROFILE_ALGORITHM_ITEMS,
default="POISSON",
)
@@ -210,14 +211,17 @@ class AnyImagePreferences(bpy.types.AddonPreferences):
default=DEFAULT_BACKGROUND_MODEL_KEY,
)
geometry_model: bpy.props.EnumProperty(
name="Generate Depth Map Model",
description="Model used to generate depth maps",
name="Depth & Normal Model",
description="Choose the AI model used to generate depth and surface normals.",
items=geometry_model_items,
default=0,
)
geometry_resolution_level: bpy.props.IntProperty(
name="MoGe Resolution Level",
description="MoGe-2 inference resolution used for geometry and normal generation",
description=(
"Set the processing resolution for depth and normal generation. "
"Higher levels can capture finer details but take more time and memory."
),
default=5,
min=0,
max=9,

66
src/anyimage/properties.py
View File

@@ -140,6 +140,18 @@ def upscale_model_label(model_key):
return model.label if model is not None else model_key
def cutout_gesture_property():
return bpy.props.EnumProperty(
name="Gesture",
description="Shape used to select the Cutout area",
items=(
("LASSO", "Lasso", "Draw a closed freehand area"),
("POLYLINE", "Polyline", "Click points to define a closed area"),
),
default="LASSO",
)
def mask_gesture_property():
return bpy.props.EnumProperty(
name="Gesture",
@@ -156,19 +168,31 @@ def mask_gesture_property():
def mask_mode_property():
return bpy.props.EnumProperty(
name="Mode",
description="How the gesture changes image Alpha",
description="Choose how the selected or painted area changes image visibility.",
items=(
("SET", "Set", "Keep current Alpha inside the gesture", "SELECT_SET", 0),
("ADD", "Add", "Extend Alpha with the gesture", "SELECT_EXTEND", 1),
(
"SET",
"Set",
"Keep visible content inside the selected or painted area and hide everything outside.",
"SELECT_SET",
0,
),
(
"EXTEND",
"Extend",
"Reveal the selected or painted area while keeping other areas unchanged.",
"SELECT_EXTEND",
1,
),
(
"SUBTRACT",
"Subtract",
"Remove Alpha inside the gesture",
"Hide the selected or painted area while keeping other areas unchanged.",
"SELECT_SUBTRACT",
2,
),
),
default="SUBTRACT",
default="SET",
)
@@ -183,24 +207,12 @@ def mask_radius_property():
)
def cutout_inward_axis_property():
return bpy.props.EnumProperty(
name="Inward Axis",
description="Local axis pointing inward from the image sheet",
items=(
("NEGATIVE_Z", "-Z", "Match the source Image Empty local axes"),
("POSITIVE_X", "+X", "Use the original Cutout local axes"),
),
default="NEGATIVE_Z",
)
def cutout_generate_normal_property():
return bpy.props.BoolProperty(
name="Normal Map",
description=(
"Generate a normal map for Flat and Solid; "
"Depth Solid and Depth Balloon always generate normal maps"
"Generate a normal map to add surface detail under lighting for Flat and Solid shapes. "
"Depth Solid and Depth Balloon always generate normal maps."
),
default=False,
)
@@ -209,7 +221,10 @@ def cutout_generate_normal_property():
def cutout_alpha_threshold_property():
return bpy.props.FloatProperty(
name="Alpha Threshold",
description="Minimum Alpha kept in the Cutout outline",
description=(
"Exclude areas below this opacity from the Cutout outline. "
"Higher values exclude more semi-transparent edges."
),
default=0.9,
min=0.0,
max=1.0,
@@ -230,7 +245,7 @@ def cutout_fine_outline_property():
def cutout_mesh_detail_property():
return bpy.props.EnumProperty(
name="Mesh Detail",
description="Mesh sampling density",
description="Control mesh density. Higher levels use more vertices to represent finer shapes.",
items=(
("LOW", "Low", "32 source image pixels between neighboring mesh points"),
("MEDIUM", "Medium", "16 source image pixels between neighboring mesh points"),
@@ -242,18 +257,11 @@ def cutout_mesh_detail_property():
class AnyImageSettings(bpy.types.PropertyGroup):
cutout_gesture: cutout_gesture_property()
cutout_alpha_threshold: cutout_alpha_threshold_property()
cutout_mesh_detail: cutout_mesh_detail_property()
cutout_fine_outline: cutout_fine_outline_property()
cutout_generate_normal: cutout_generate_normal_property()
cutout_inward_axis: cutout_inward_axis_property()
mask_gesture: mask_gesture_property()
mask_mode: mask_mode_property()
mask_radius: mask_radius_property()
video_frames: bpy.props.IntProperty(
name="Extract Frames",
description="Maximum number of consecutive frames extracted from the start of a video or frame sequence",
default=16,
min=1,
soft_max=64,
)

2
src/anyimage/runtime.py
View File

@@ -12,7 +12,7 @@ ENVIRONMENT = {
"numpy==2.4.2",
"scipy==1.15.3",
"pillow==12.1.1",
"av==17.0.0",
"trimesh==4.11.2",
"huggingface-hub==1.4.1",
],

7
src/anyimage/server/geometry/prediction_artifacts.py
View File

@@ -64,16 +64,13 @@ def generate_moge_artifacts(
parameters.get("max_input_size", DEFAULT_MAX_AI_INPUT_SIZE),
)
try:
frames = moge.infer(
frame = moge.infer(
context.resource("model_manager"),
parameters,
[model_path],
model_path,
include_points=generate_depth,
cancel_check=context.check_cancelled,
)
if len(frames) != 1:
raise ValueError("Artifact generation requires one geometry frame")
frame = frames[0]
result = {}
if generate_depth:
context.progress(0.45, "Generating depth texture")

4
src/anyimage/server/jobs/cutout.py
View File

@@ -1,6 +1,6 @@
"""Generate optional AI files from one rectangular Cutout image input."""
from pathlib import Path
from ..media.input import validate_input_path
from ..geometry.prediction_artifacts import generate_moge_artifacts
@@ -8,7 +8,7 @@ from ..geometry.prediction_artifacts import generate_moge_artifacts
def run(context, parameters):
context.progress(0.05, "Preparing Cutout Selection")
context.directory.mkdir(parents=True, exist_ok=True)
input_path = Path(parameters["input"])
input_path = validate_input_path(parameters["input"])
result = {}
generate_depth = bool(parameters.get("generate_depth", False))
normal_mode = str(parameters.get("normal_mode", "NONE"))

20
src/anyimage/server/jobs/depth_plane.py
View File

@@ -1,7 +1,7 @@
"""Generate model-space depth and normals for one complete image."""
from ..geometry.prediction_artifacts import generate_moge_artifacts
from ..media import input_frames
from ..media.input import validate_input_path
def run(context, parameters):
@@ -10,16 +10,14 @@ def run(context, parameters):
raise ValueError(f"Unsupported plane type: {plane_type}")
context.progress(0.05, "Preparing depth conversion")
context.directory.mkdir(parents=True, exist_ok=True)
with input_frames(context, parameters) as images:
if len(images) != 1:
raise ValueError("Depth conversion currently supports one image")
artifacts = generate_moge_artifacts(
context,
parameters,
images[0],
generate_depth=True,
normal_mode="OBJECT" if plane_type == "DEPTH" else "TANGENT",
)
input_path = validate_input_path(parameters["input"])
artifacts = generate_moge_artifacts(
context,
parameters,
input_path,
generate_depth=True,
normal_mode="OBJECT" if plane_type == "DEPTH" else "TANGENT",
)
context.check_cancelled()
context.progress(1.0, "Depth conversion data is ready")
return {key: path.name for key, path in artifacts.items()}

103
src/anyimage/server/jobs/panorama.py
View File

@@ -1,12 +1,10 @@
"""Generate a radial depth texture from the current RGBA panorama."""
import json
from pathlib import Path
from ..geometry.depth_texture import write_float_exr
from ..geometry.panorama import camera_bases, merge_distances, sample_panorama, view_rays
from ..media import VIDEO_SUFFIXES, input_frames
from ..media.input import open_image
from ..media.input import open_image, validate_input_path
from ..media.resolution import DEFAULT_MAX_AI_INPUT_SIZE, normalized_max_input_size
from ..models.moge import infer_image as infer
@@ -15,63 +13,58 @@ def run(context, parameters):
import numpy as np
context.check_cancelled()
input_path = Path(parameters["input"])
if input_path.is_dir() or input_path.suffix.lower() in VIDEO_SUFFIXES:
raise ValueError("Panorama conversion requires one static 2:1 image")
input_path = validate_input_path(parameters["input"])
maximum = normalized_max_input_size(parameters.get("max_input_size", DEFAULT_MAX_AI_INPUT_SIZE))
context.directory.mkdir(parents=True, exist_ok=True)
outputs = [context.directory / name for name in ("depth.exr", "depth.json")]
try:
with input_frames(context, parameters) as images:
if len(images) != 1:
with open_image(input_path) as opened:
if opened.width != opened.height * 2:
raise ValueError("Panorama conversion requires one static 2:1 image")
with open_image(images[0]) as opened:
if getattr(opened, "n_frames", 1) != 1 or opened.width != opened.height * 2:
raise ValueError("Panorama conversion requires one static 2:1 image")
source = opened.convert("RGBA")
width, height = source.size
size = min(maximum, max(512, width // 4))
image = np.asarray(source, dtype=np.float32) / 255.0
manager = context.resource("model_manager")
bases = camera_bases()
rays = view_rays(size)
distances, masks = [], []
session = None
for index, basis in enumerate(bases):
context.check_cancelled()
context.progress(0.05 + 0.65 * index / len(bases), f"Generating panorama view {index + 1}/12")
directions = rays @ basis
alpha = sample_panorama(image[..., 3], directions, order=0)
visible = alpha > 0
if not visible.any():
distances.append(np.ones((size, size), dtype=np.float32))
masks.append(visible)
continue
if session is None:
session = manager.get_moge(manager.directory(parameters["model"]), parameters["device"])
color = sample_panorama(image[..., :3], directions) * alpha[..., None]
prediction = infer(session, color, parameters["resolution_level"],
fov_x=90.0, source_valid=visible)
distances.append(np.linalg.norm(prediction["points"], axis=-1))
masks.append((prediction["mask"] > 0.5) & visible)
del prediction
context.progress(0.75, "Merging panoramic distance")
distance, valid = merge_distances(distances, masks, bases, width, height,
cancel_check=context.check_cancelled)
alpha = np.asarray(source.getchannel("A"),
dtype=np.float32) / 255.0
alpha *= valid
valid_depth = (alpha >= 0.9) & np.isfinite(distance) & (distance > 0)
if not valid_depth.any():
raise ValueError("The panorama contains no valid depth pixels")
if not np.isfinite(distance).all() or (distance <= 0).any():
raise ValueError("Panorama fusion produced invalid distances")
rgba = np.empty((height, width, 4), dtype=np.float32)
rgba[..., :3], rgba[..., 3] = distance[..., None], alpha
write_float_exr(rgba, outputs[0])
outputs[1].write_text(json.dumps({
"projection": "equirectangular", "image_size": [width, height],
}, separators=(",", ":")), encoding="utf-8")
source = opened.convert("RGBA")
width, height = source.size
size = min(maximum, max(512, width // 4))
image = np.asarray(source, dtype=np.float32) / 255.0
manager = context.resource("model_manager")
bases = camera_bases()
rays = view_rays(size)
distances, masks = [], []
session = None
for index, basis in enumerate(bases):
context.check_cancelled()
context.progress(0.05 + 0.65 * index / len(bases), f"Generating panorama view {index + 1}/12")
directions = rays @ basis
alpha = sample_panorama(image[..., 3], directions, order=0)
visible = alpha > 0
if not visible.any():
distances.append(np.ones((size, size), dtype=np.float32))
masks.append(visible)
continue
if session is None:
session = manager.get_moge(manager.directory(parameters["model"]), parameters["device"])
color = sample_panorama(image[..., :3], directions) * alpha[..., None]
prediction = infer(session, color, parameters["resolution_level"],
fov_x=90.0, source_valid=visible)
distances.append(np.linalg.norm(prediction["points"], axis=-1))
masks.append((prediction["mask"] > 0.5) & visible)
del prediction
context.progress(0.75, "Merging panoramic distance")
distance, valid = merge_distances(distances, masks, bases, width, height,
cancel_check=context.check_cancelled)
alpha = np.asarray(source.getchannel("A"),
dtype=np.float32) / 255.0
alpha *= valid
valid_depth = (alpha >= 0.9) & np.isfinite(distance) & (distance > 0)
if not valid_depth.any():
raise ValueError("The panorama contains no valid depth pixels")
if not np.isfinite(distance).all() or (distance <= 0).any():
raise ValueError("Panorama fusion produced invalid distances")
rgba = np.empty((height, width, 4), dtype=np.float32)
rgba[..., :3], rgba[..., 3] = distance[..., None], alpha
write_float_exr(rgba, outputs[0])
outputs[1].write_text(json.dumps({
"projection": "equirectangular", "image_size": [width, height],
}, separators=(",", ":")), encoding="utf-8")
context.check_cancelled()
context.progress(1.0, "Panorama conversion data is ready")
return {"depth": outputs[0].name, "depth_metadata": outputs[1].name}

84
src/anyimage/server/jobs/remove_background.py
View File

@@ -1,9 +1,8 @@
import time
from ..media import input_frames
from ..media.input import open_image
from ..media.input import open_image, validate_input_path
from ..model_catalog import BACKGROUND_MODELS
from ..models.background import infer_alpha, infer_foreground, infer_foreground_frames, inference_device
from ..models.background import infer_alpha, infer_foreground, inference_device
def run(context, parameters):
@@ -24,62 +23,45 @@ def run(context, parameters):
if output_kind not in {"foreground", "alpha"}:
raise ValueError(f"Unknown background removal output: {output_kind}")
with input_frames(context, parameters) as images:
context.progress(0.12, f"Loading {parameters['model']} for {device.upper()}")
if output_kind == "alpha":
import numpy as np
input_path = validate_input_path(parameters["input"])
context.progress(0.12, f"Loading {parameters['model']} for {device.upper()}")
if output_kind == "alpha":
import numpy as np
if len(images) != 1:
raise ValueError("Alpha output requires one still image")
with open_image(images[0]) as source:
alpha, load_ms, inference_ms = infer_alpha(
source, model_directory, device, model_cache=model_manager,
cancel_check=context.check_cancelled,
)
context.check_cancelled()
np.save(output_directory / "alpha.npy", alpha, allow_pickle=False)
elif len(images) == 1:
context.progress(0.35, "Removing background")
foreground, load_ms, inference_ms = infer_foreground(
images[0],
model_directory,
device,
model_cache=model_manager,
with open_image(input_path) as source:
alpha, load_ms, inference_ms = infer_alpha(
source, model_directory, device, model_cache=model_manager,
cancel_check=context.check_cancelled,
preserve_input_alpha=True,
)
context.check_cancelled()
context.progress(0.92, "Writing transparent PNG")
output_path = output_directory / "image.0001.png"
foreground.convert("RGBA").save(
output_path,
format="PNG",
compress_level=1,
)
if not output_path.is_file():
raise RuntimeError("Background removal did not write a foreground image")
else:
load_ms, inference_ms = infer_foreground_frames(
images,
model_directory,
device,
output_directory,
context.progress,
0.12,
0.7,
model_cache=model_manager,
cancel_check=context.check_cancelled,
output_prefix="image",
preserve_input_alpha=True,
)
context.check_cancelled()
np.save(output_directory / "alpha.npy", alpha, allow_pickle=False)
else:
context.progress(0.35, "Removing background")
foreground, load_ms, inference_ms = infer_foreground(
input_path,
model_directory,
device,
model_cache=model_manager,
cancel_check=context.check_cancelled,
preserve_input_alpha=True,
)
context.check_cancelled()
context.progress(0.92, "Writing transparent PNG")
output_path = output_directory / "foreground.png"
foreground.convert("RGBA").save(
output_path,
format="PNG",
compress_level=1,
)
if not output_path.is_file():
raise RuntimeError("Background removal did not write a foreground image")
context.check_cancelled()
total_ms = (time.perf_counter() - total_started) * 1000.0
frame_text = "" if len(images) == 1 else f" from {len(images)} frame(s)"
message = (
f"Background removed{frame_text} in {inference_ms:.0f} ms "
f"Background removed in {inference_ms:.0f} ms "
f"(model load {load_ms:.0f} ms, total {total_ms:.0f} ms)"
)
context.log(message)
context.progress(1.0, message)
return {"alpha": "alpha.npy"} if output_kind == "alpha" else {"image": "."}
return {"alpha": "alpha.npy"} if output_kind == "alpha" else {"foreground": "foreground.png"}

43
src/anyimage/server/jobs/upscale.py
View File

@@ -1,13 +1,8 @@
from ..media import DEFAULT_MAX_AI_INPUT_SIZE, input_frames
from ..media import DEFAULT_MAX_AI_INPUT_SIZE
from ..media.input import validate_input_path
from ..models.upscale import infer_one
def frame_progress_message(message, index, total):
if total > 1:
return f"{message} ({index + 1}/{total})"
return message
def run(context, parameters):
context.log(
f"Upscale model: {parameters['model']}; "
@@ -15,27 +10,17 @@ def run(context, parameters):
f"{parameters.get('max_input_size', DEFAULT_MAX_AI_INPUT_SIZE)} px"
)
context.progress(0.02, "Preparing upscale input")
context.directory.mkdir(parents=True, exist_ok=True)
with input_frames(context, parameters) as images:
total = len(images)
for index, image in enumerate(images):
context.check_cancelled()
context.progress(
0.08 + index / total * 0.84,
frame_progress_message("Upscaling image", index, total),
)
prediction = infer_one(
context.resource("model_manager"),
parameters,
image,
cancel_check=context.check_cancelled,
release_memory=index == total - 1,
)
prediction.save(
context.directory / f"upscale.{index + 1:04d}.png",
format="PNG",
compress_level=1,
)
context.check_cancelled()
input_path = validate_input_path(parameters["input"])
context.directory.mkdir(parents=True, exist_ok=True)
context.progress(0.08, "Upscaling image")
prediction = infer_one(
context.resource("model_manager"),
parameters,
input_path,
cancel_check=context.check_cancelled,
)
context.check_cancelled()
prediction.save(context.directory / "upscale.png", format="PNG", compress_level=1)
context.progress(1.0, "Image upscale complete")
return {"upscale": "."}
return {"upscale": "upscale.png"}

14
src/anyimage/server/media/__init__.py
View File

@@ -1,14 +1,7 @@
from .input import (
IMAGE_SUFFIXES,
VIDEO_SUFFIXES,
TaskCancelled,
analysis_frames,
check_cancelled,
collect_images,
extract_video_frames,
first_frames,
input_frames,
prepare_input,
)
@@ -23,15 +16,8 @@ from .resolution import (
__all__ = (
"IMAGE_SUFFIXES",
"VIDEO_SUFFIXES",
"TaskCancelled",
"analysis_frames",
"check_cancelled",
"collect_images",
"extract_video_frames",
"first_frames",
"input_frames",
"prepare_input",
"DEFAULT_MAX_AI_INPUT_SIZE",
"MAX_MAX_AI_INPUT_SIZE",
"MIN_MAX_AI_INPUT_SIZE",

168
src/anyimage/server/media/input.py
View File

@@ -1,46 +1,21 @@
import shutil
import tempfile
from contextlib import contextmanager
from pathlib import Path
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp"}
VIDEO_SUFFIXES = {
".mp4",
".mov",
".avi",
".mkv",
".webm",
".m4v",
".mts",
".mpg",
".mpeg",
}
def validate_input_path(value, *, images_only=False):
"""Validate supported media without importing image or video libraries."""
def validate_input_path(value):
"""Validate one supported image file without importing image libraries."""
path = Path(value)
if path.is_dir() and not images_only:
if not collect_images(path):
supported = ", ".join(sorted(IMAGE_SUFFIXES))
raise RuntimeError(
f"No supported image frames found in: {path}. Supported images: {supported}."
)
return path
if not path.is_file():
kind = "video" if path.suffix.lower() in VIDEO_SUFFIXES else "image"
raise RuntimeError(f"Source {kind} file does not exist: {path}")
raise RuntimeError(f"Source image file does not exist: {path}")
suffix = path.suffix.lower()
supported = IMAGE_SUFFIXES if images_only else IMAGE_SUFFIXES | VIDEO_SUFFIXES
if suffix not in supported:
message = (
if suffix not in IMAGE_SUFFIXES:
raise RuntimeError(
f"Unsupported input format: {suffix or '(no extension)'}. "
f"Supported images: {', '.join(sorted(IMAGE_SUFFIXES))}."
)
if not images_only:
message += f" Supported videos: {', '.join(sorted(VIDEO_SUFFIXES))}."
raise RuntimeError(message)
return path
@@ -54,6 +29,8 @@ def open_image(path):
try:
opened = Image.open(path)
opened.load()
if getattr(opened, "n_frames", 1) != 1:
raise RuntimeError("Only still images are supported")
except (OSError, ValueError, SyntaxError) as error:
raise RuntimeError(f"Unable to read image: {path}") from error
yield opened
@@ -69,134 +46,3 @@ class TaskCancelled(RuntimeError):
def check_cancelled(cancel_check):
if cancel_check is not None and cancel_check():
raise TaskCancelled("Task cancelled")
def collect_images(directory):
directory = Path(directory)
return sorted(
path
for path in directory.iterdir()
if path.is_file() and path.suffix.lower() in IMAGE_SUFFIXES
)
def first_frames(images, maximum_frames, start_frame=1):
"""Return up to maximum_frames image paths in order from start_frame."""
if maximum_frames <= 0:
return images
start_index = max(start_frame - 1, 0)
return images[start_index:start_index + maximum_frames]
def extract_video_frames(
video_path,
output_directory,
maximum_frames,
av,
start_frame=1,
cancel_check=None,
):
output_directory = Path(output_directory)
output_directory.mkdir(parents=True, exist_ok=True)
try:
container = av.open(str(video_path))
except Exception as error:
raise RuntimeError(f"Unable to read video: {video_path}") from error
image_paths = []
skipped = 0
try:
try:
stream = container.streams.video[0]
frames = iter(container.decode(stream))
except Exception as error:
raise RuntimeError(f"Unable to read video: {video_path}") from error
while True:
try:
frame = next(frames)
except StopIteration:
break
except Exception as error:
raise RuntimeError(f"Unable to read video: {video_path}") from error
check_cancelled(cancel_check)
if skipped < start_frame - 1:
skipped += 1
continue
image_path = output_directory / f"frame.{len(image_paths) + 1:04d}.jpg"
try:
decoded = frame.to_image()
except Exception as error:
raise RuntimeError(f"Unable to read video: {video_path}") from error
decoded.save(image_path, format="JPEG")
image_paths.append(image_path)
if len(image_paths) >= maximum_frames:
break
finally:
container.close()
if not image_paths:
raise RuntimeError(f"Unable to read video: {video_path}")
return image_paths
def prepare_input(
input_path,
video_frames,
temporary_root,
start_frame=1,
cancel_check=None,
):
check_cancelled(cancel_check)
input_path = validate_input_path(input_path)
if input_path.is_dir():
images = collect_images(input_path)
return first_frames(images, video_frames, start_frame), None
suffix = input_path.suffix.lower()
if suffix in IMAGE_SUFFIXES:
return [input_path], None
import av
temporary_directory = Path(
tempfile.mkdtemp(prefix="anyimage-video-", dir=temporary_root)
)
try:
images = extract_video_frames(
input_path,
temporary_directory,
video_frames,
av,
start_frame=start_frame,
cancel_check=cancel_check,
)
except Exception:
shutil.rmtree(temporary_directory, ignore_errors=True)
raise
return images, temporary_directory
@contextmanager
def input_frames(context, parameters):
"""Yield prepared input frames and clean up extracted video frames."""
images, temporary_directory = prepare_input(
parameters["input"],
parameters.get("video_frames", 16),
context.directory,
start_frame=parameters.get("start_frame", 1),
cancel_check=context.check_cancelled,
)
try:
yield images
finally:
if temporary_directory is not None:
shutil.rmtree(temporary_directory, ignore_errors=True)
@contextmanager
def analysis_frames(context, parameters, fallback_images):
"""Yield model-analysis frames, falling back to the color input."""
analysis_input = parameters.get("analysis_input")
if not analysis_input or analysis_input == parameters["input"]:
yield fallback_images
return
with input_frames(context, dict(parameters, input=analysis_input)) as images:
yield images

24
src/anyimage/server/model_catalog.py
View File

@@ -22,7 +22,7 @@ BIREFNET_LITE = ModelSpec(
directory_name="birefnet-lite",
repository="ZhengPeng7/BiRefNet_lite",
license_name="MIT",
description="Compact 1024px background removal with continuous alpha",
description="Remove backgrounds at 1024 px processing resolution while preserving soft, semi-transparent edges.",
download_patterns=("model.onnx", "LICENSE.txt"),
ready_patterns=("model.onnx", "LICENSE.txt"),
r2_directory="birefnet-lite",
@@ -40,7 +40,7 @@ BEN2_MODEL = ModelSpec(
repository="onnx-community/BEN2-ONNX",
license_name="Apache 2.0",
huggingface_repository="onnx-community/BEN2-ONNX",
description="General-purpose FP16 ONNX background removal model",
description="Remove backgrounds from general images while preserving foreground transparency.",
download_patterns=("onnx/model_fp16.onnx",),
ready_patterns=("onnx/model_fp16.onnx",),
r2_directory="BEN2-ONNX",
@@ -58,7 +58,7 @@ REALESRGAN_GENERAL_WDN_X4V3 = ModelSpec(
directory_name="realesr-general-wdn-x4v3",
repository="xinntao/Real-ESRGAN",
license_name="BSD 3-Clause",
description="Fast and conservative 2x output from a 4x ONNX model",
description="Enlarge images to twice their width and height with fast, restrained detail enhancement.",
download_patterns=("realesr-general-wdn-x4v3.onnx",),
ready_patterns=("realesr-general-wdn-x4v3.onnx",),
r2_directory="realesr-general-wdn-x4v3",
@@ -77,7 +77,7 @@ REALESRGAN_X4PLUS = ModelSpec(
repository="jonathanst29/tinier-upscale-models",
license_name="BSD 3-Clause",
huggingface_repository="jonathanst29/tinier-upscale-models",
description="Detailed 2x output from a general-purpose 4x ONNX model",
description="Enlarge images to twice their width and height with stronger detail enhancement.",
download_patterns=("realesrgan-x4plus.onnx",),
ready_patterns=("realesrgan-x4plus.onnx",),
r2_directory="realesrgan-x4plus",
@@ -97,7 +97,10 @@ HAT_GAN_X4_SHARPER = ModelSpec(
directory_name="hat-gan-x4-sharper",
repository="XPixelGroup/HAT",
license_name="Apache 2.0",
description="Detailed texture enhancement with 2x output; higher processing cost",
description=(
"Enlarge images to twice their width and height with detailed texture enhancement. "
"Requires more processing time."
),
download_patterns=("model.onnx", "LICENSE.txt"),
ready_patterns=("model.onnx", "LICENSE.txt"),
r2_directory="hat-gan-x4-sharper",
@@ -129,7 +132,7 @@ MOGE2_MODELS = {
repository="Ruicheng/moge-2-vits-normal-onnx",
license_name="MIT",
huggingface_repository="Ruicheng/moge-2-vits-normal-onnx",
description="Fastest MoGe-2 model using ONNX Runtime",
description="Generate depth and surface normals with the fastest MoGe-2 option.",
download_patterns=("model.onnx",),
ready_patterns=("model.onnx",),
r2_directory="moge-2-vits-normal-onnx",
@@ -146,7 +149,7 @@ MOGE2_MODELS = {
repository="Ruicheng/moge-2-vitb-normal-onnx",
license_name="MIT",
huggingface_repository="Ruicheng/moge-2-vitb-normal-onnx",
description="Balanced MoGe-2 model using ONNX Runtime",
description="Generate depth and surface normals with a balance of processing speed and detail.",
download_patterns=("model.onnx",),
ready_patterns=("model.onnx",),
r2_directory="moge-2-vitb-normal-onnx",
@@ -163,7 +166,7 @@ MOGE3_VITL = ModelSpec(
directory_name="moge-3-vitl-onnx",
repository="Ruicheng/moge-3-vitl",
license_name="MIT",
description="MoGe-3 geometry with three refinement steps using ONNX Runtime",
description="Generate depth and surface normals with MoGe-3, refining the prediction in three steps.",
download_patterns=("backbone.onnx", "refiner.onnx"),
ready_patterns=("backbone.onnx", "refiner.onnx"),
r2_directory="moge-3-vitl-onnx",
@@ -180,7 +183,10 @@ BIREFNET_HR_MATTING = ModelSpec(
directory_name="birefnet-hr-matting",
repository="ZhengPeng7/BiRefNet_HR-matting",
license_name="MIT",
description="2048px matting with continuous alpha; CPU only, high memory and processing cost",
description=(
"Remove backgrounds at 2048 px processing resolution while preserving soft, semi-transparent edges. "
"Runs on CPU and requires more time and memory."
),
download_patterns=("model.onnx", "LICENSE.txt"),
ready_patterns=("model.onnx", "LICENSE.txt"),
r2_directory="birefnet-hr-matting",

46
src/anyimage/server/models/background.py
View File

@@ -75,7 +75,7 @@ def infer_foreground(
from PIL import Image, ImageChops
check_cancelled(cancel_check)
input_path = validate_input_path(Path(input_path), images_only=True)
input_path = validate_input_path(Path(input_path))
with open_image(input_path) as opened_image:
source = opened_image.convert("RGBA")
alpha, load_ms, inference_ms = infer_alpha(
@@ -91,47 +91,3 @@ def infer_foreground(
foreground.paste(source.convert("RGB"), (0, 0), source.getchannel("A"))
foreground.putalpha(mask)
return foreground, load_ms, inference_ms
def infer_foreground_frames(
images,
model_dir,
device,
output_directory,
progress_callback,
start_progress,
progress_span,
model_cache=None,
cancel_check=None,
output_prefix="foreground",
preserve_input_alpha=False,
):
"""Remove each frame's background and write numbered transparent PNG files."""
total_load_ms = 0.0
total_inference_ms = 0.0
for index, image_path in enumerate(images):
check_cancelled(cancel_check)
progress_callback(
start_progress + progress_span * index / len(images),
f"Removing background ({index + 1}/{len(images)})",
)
foreground, load_ms, inference_ms = infer_foreground(
image_path,
model_dir,
device,
model_cache=model_cache,
cancel_check=cancel_check,
preserve_input_alpha=preserve_input_alpha,
release_memory=index == len(images) - 1,
)
output_path = output_directory / f"{output_prefix}.{index + 1:04d}.png"
foreground.convert("RGBA").save(
output_path,
format="PNG",
compress_level=1,
)
if not output_path.is_file():
raise RuntimeError(f"Background removal did not write: {output_path}")
total_load_ms += load_ms
total_inference_ms += inference_ms
return total_load_ms, total_inference_ms

26
src/anyimage/server/models/moge.py
View File

@@ -38,30 +38,16 @@ def _geometry_frame(prediction, include_points):
def infer(
model_manager,
parameters,
images,
image_path,
*,
include_points,
progress_callback=None,
cancel_check=None,
):
frames = []
for index, image_path in enumerate(images):
if cancel_check is not None:
cancel_check()
if progress_callback is not None:
progress_callback(index, len(images))
frames.append(
_geometry_frame(
infer_one(
model_manager,
parameters,
image_path,
release_memory=index == len(images) - 1,
),
include_points,
)
)
return frames
"""Infer one image and return its geometry fields."""
if cancel_check is not None:
cancel_check()
prediction = infer_one(model_manager, parameters, image_path)
return _geometry_frame(prediction, include_points)
def infer_one(