import re import shutil from pathlib import Path import bpy DEPTH_KEY_CANDIDATES = ("depth", "depths") INTRINSICS_KEY_CANDIDATES = ("intrinsics", "ixts") DISPLACEMENT_STRENGTH = -1.0 DISPLACEMENT_MIDLEVEL = 1.0 ORIGINAL_DISPLACEMENT_STRENGTH = 1.0 ORIGINAL_DISPLACEMENT_MIDLEVEL = 0.0 BUMP_BASE_DISTANCE = 0.001 IMAGE_NAME_SUFFIXES = ( ".jpeg", ".jpg", ".png", ".webp", ".tiff", ".tif", ".exr", ".hdr", ".bmp", ".tga", ) VIDEO_NAME_SUFFIXES = ( ".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".mts", ".mpg", ".mpeg", ) def image_depth_model_items(owner, context): from .models import model_enum_items from .runtime import model_ready return [ item for item in model_enum_items(owner, context) if model_ready(item[0]) ] _last_depth_plane_model = "BASE" def is_image_empty(candidate): return bool( candidate and getattr(candidate, "type", None) == "EMPTY" and getattr(candidate, "empty_display_type", None) == "IMAGE" and getattr(candidate, "data", None) is not None ) def is_animated_image(image): return getattr(image, "source", None) in {"MOVIE", "SEQUENCE"} def video_total_frames(image): """Return the total animated frame count, or 1 for a still image.""" source = getattr(image, "source", None) if source == "MOVIE": return int(getattr(image, "frame_duration", 1) or 1) if source == "SEQUENCE": return len(collect_sequence_frames(image)) or 1 return 1 def image_user_settings(source_object): """Return the Empty's image user settings as a four-item tuple.""" image_user = getattr(source_object, "image_user", None) if image_user is None: return 1, 0, 0, False return ( int(getattr(image_user, "frame_start", 1) or 1), int(getattr(image_user, "frame_offset", 0) or 0), int(getattr(image_user, "frame_duration", 0) or 0), bool(getattr(image_user, "use_cyclic", False)), ) def extract_frame_range(source_object): """Return the clamped (start_frame, frame_count) range extracted from the source.""" total_frames = video_total_frames(getattr(source_object, "data", None)) if total_frames <= 1: return 1, total_frames _frame_start, frame_offset, image_frames, _use_cyclic = ( image_user_settings(source_object) ) frame_count = ( min(image_frames, total_frames) if image_frames > 1 else total_frames ) start_frame = min(1 + frame_offset, total_frames - frame_count + 1) start_frame = max(start_frame, 1) return start_frame, frame_count def default_extract_frames(source_object): """Return the Empty's Image Frames value, or the full frame count when unset.""" return extract_frame_range(source_object)[1] def image_path(image): filepath_from_user = getattr(image, "filepath_from_user", None) filepath = ( filepath_from_user() if callable(filepath_from_user) else getattr(image, "filepath", "") ) if not filepath: return Path() return Path(bpy.path.abspath(filepath)).expanduser() def image_is_packed(image): if getattr(image, "packed_file", None) is not None: return True return bool(getattr(image, "packed_files", ())) def remove_temporary_input(path): """Remove a temporary input file or frame directory.""" path = Path(path) try: if path.is_dir(): shutil.rmtree(path, ignore_errors=True) else: path.unlink(missing_ok=True) except OSError: pass def image_base_name(image): name = image.name lower_name = name.lower() for suffix in IMAGE_NAME_SUFFIXES + VIDEO_NAME_SUFFIXES: suffix_index = lower_name.rfind(suffix) if suffix_index <= 0: continue trailing = name[suffix_index + len(suffix) :] if trailing and not ( trailing.startswith(".") and trailing[1:].isdigit() ): continue return f"{name[:suffix_index]}{trailing}" return name def depth_data_name(source_object): return f"{image_base_name(source_object.data)}.depth" def copy_depth_image(source_object): """Return a renamed copy of the source image for use as the depth map.""" depth_image = source_object.data.copy() depth_image.name = depth_data_name(source_object) return depth_image def removed_background_data_name(source_object): return f"{image_base_name(source_object.data)}.foreground" def scaled_process_resolution(image, scale): width, height = image.size if width <= 0 or height <= 0: raise ValueError("The source image has no readable dimensions") if scale < 0.1 or scale > 1.0: raise ValueError("Resolution Scale must be between 0.1 and 1.0") return int(max(width, height) * scale + 0.5) def save_packed_image(image, scene): from .runtime import temporary_input_path output_path = temporary_input_path() image_settings = scene.render.image_settings original_file_format = image_settings.file_format try: image_settings.file_format = "PNG" image.save_render(str(output_path), scene=scene) except (OSError, RuntimeError) as error: output_path.unlink(missing_ok=True) raise RuntimeError("Unable to export the packed image") from error finally: image_settings.file_format = original_file_format if not output_path.is_file(): raise RuntimeError("Blender did not export the packed image") return output_path def collect_sequence_frames(image): """Return the numbered frame files of a SEQUENCE image, or [] when absent.""" source_path = image_path(image) if not is_animated_image(image) or not source_path.is_file(): return [] match = re.match( r"^(?P.*?)(?P\d+)(?P\.[^.]+)$", source_path.name, ) if match is None: return [] prefix = match.group("prefix") suffix = match.group("suffix").lower() frames = [] for path in source_path.parent.iterdir(): frame_match = re.match( r"^(?P.*?)(?P\d+)(?P\.[^.]+)$", path.name, ) if ( frame_match is not None and frame_match.group("prefix") == prefix and frame_match.group("suffix").lower() == suffix ): frames.append((int(frame_match.group("digits")), path)) return [path for _digits, path in sorted(frames)] def prepare_image_input(image, scene): source_path = image_path(image) if image_is_packed(image): return save_packed_image(image, scene), True if getattr(image, "source", None) == "MOVIE": if not source_path.is_file(): raise RuntimeError( "The image empty must reference an existing video file" ) return source_path, False sequence_frames = collect_sequence_frames(image) if sequence_frames: from .runtime import temporary_input_directory temporary_directory = Path(temporary_input_directory()) try: for index, frame_path in enumerate(sequence_frames): suffix = frame_path.suffix.lower() or ".jpg" shutil.copy2( frame_path, temporary_directory / f"frame.{index + 1:04d}{suffix}", ) except Exception: shutil.rmtree(temporary_directory, ignore_errors=True) raise return temporary_directory, True if not source_path.is_file(): raise RuntimeError( "The image empty must reference an existing image file" ) return source_path, False def configure_image_user( image_user, frame_count=None, frame_start=1, frame_offset=0, use_cyclic=None, ): """Map the image to scene frames and enable auto refresh.""" if image_user is None: return if hasattr(image_user, "frame_start"): image_user.frame_start = frame_start if hasattr(image_user, "frame_offset"): image_user.frame_offset = frame_offset if hasattr(image_user, "use_auto_refresh"): image_user.use_auto_refresh = True if use_cyclic is not None and hasattr(image_user, "use_cyclic"): image_user.use_cyclic = use_cyclic if frame_count is not None and hasattr(image_user, "frame_duration"): image_user.frame_duration = frame_count def pack_sequence_if_possible(image, expected_frames): """Pack a frame sequence when Blender supports it; otherwise keep external files.""" if expected_frames <= 0: return False try: image.pack() except (OSError, RuntimeError): return False if len(getattr(image, "packed_files", ())) >= expected_frames: return True try: image.unpack(method="REMOVE") except (OSError, RuntimeError): pass return False def sorted_frame_paths(directory, prefix=None): """Return image files in a directory sorted by their trailing frame number.""" def frame_key(path): match = re.search(r"(\d+)(?=\.[^.]+$)", path.name) return (int(match.group(1)) if match is not None else 0, path.name) return sorted( ( path for path in Path(directory).iterdir() if path.is_file() and path.suffix.lower() in IMAGE_NAME_SUFFIXES and (prefix is None or path.stem.startswith(prefix)) ), key=frame_key, ) def force_sequence_source(image): """Ensure a loaded image is a SEQUENCE datablock for its numbered frames.""" if getattr(image, "source", None) == "SEQUENCE": return image try: image.source = "SEQUENCE" except (OSError, RuntimeError, TypeError) as error: raise RuntimeError( "Blender cannot load the frame sequence" ) from error if getattr(image, "source", None) != "SEQUENCE": raise RuntimeError("Blender cannot load the frame sequence") return image def create_sequence_image(name, frame_directory, frame_prefix="frame."): """Load a numbered frame directory as a SEQUENCE image datablock.""" frame_paths = sorted_frame_paths(frame_directory, frame_prefix) if not frame_paths: raise RuntimeError(f"No image frames found in: {frame_directory}") try: image = bpy.data.images.load( str(frame_paths[0]), check_existing=False, ) force_sequence_source(image) except (OSError, RuntimeError) as error: if "image" in locals(): bpy.data.images.remove(image, do_unlink=True) raise RuntimeError("Unable to load the frame sequence") from error image.name = name pack_sequence_if_possible(image, len(frame_paths)) return image def create_depth_sequence_image(name, depth_frames, scene, frame_directory): """Write one EXR per depth frame and return a SEQUENCE image datablock.""" import numpy as np frame_directory = Path(frame_directory) frame_directory.mkdir(parents=True, exist_ok=True) height, width = depth_frames[0].shape image = bpy.data.images.new( name, width=width, height=height, alpha=False, float_buffer=True, ) image.colorspace_settings.name = "Non-Color" image_settings = scene.render.image_settings original_file_format = image_settings.file_format frame_paths = [] try: image_settings.file_format = "OPEN_EXR" for frame_index, depth_values in enumerate(depth_frames): # NPZ arrays use a top-left origin, while Blender image pixels start below. depth_pixels = np.flipud(depth_values) rgba = np.empty((height, width, 4), dtype=np.float32) rgba[..., 0] = depth_pixels rgba[..., 1] = depth_pixels rgba[..., 2] = depth_pixels rgba[..., 3] = 1.0 image.pixels.foreach_set(rgba.ravel()) frame_path = ( frame_directory / f"depth.{frame_index + 1:04d}.exr" ) image.save_render(str(frame_path), scene=scene) if not frame_path.is_file(): raise RuntimeError(f"Blender did not write: {frame_path}") frame_paths.append(frame_path) finally: image_settings.file_format = original_file_format bpy.data.images.remove(image, do_unlink=True) if not frame_paths: raise RuntimeError("Unable to write the depth frame sequence") try: sequence_image = bpy.data.images.load( str(frame_paths[0]), check_existing=False, ) force_sequence_source(sequence_image) except (OSError, RuntimeError) as error: if "sequence_image" in locals(): bpy.data.images.remove(sequence_image, do_unlink=True) raise RuntimeError( "Unable to load the depth frame sequence" ) from error sequence_image.name = name sequence_image.colorspace_settings.name = "Non-Color" pack_sequence_if_possible(sequence_image, len(frame_paths)) return sequence_image def create_removed_background_image( source_object, output_path, assign_to_source=True, ): output_path = Path(output_path) if output_path.is_dir(): foreground_paths = sorted_frame_paths(output_path, "foreground.") if not foreground_paths: raise RuntimeError( f"Background removal result has no foreground frames: " f"{output_path}" ) try: result_image = bpy.data.images.load( str(foreground_paths[0]), check_existing=False, ) force_sequence_source(result_image) except (OSError, RuntimeError) as error: if "result_image" in locals(): bpy.data.images.remove(result_image, do_unlink=True) raise RuntimeError("Unable to load the background result") from error source_image = source_object.data original_fake_user = source_image.use_fake_user try: result_image.name = removed_background_data_name(source_object) pack_sequence_if_possible(result_image, len(foreground_paths)) if assign_to_source: image_user = getattr(source_object, "image_user", None) ( original_frame_start, _original_frame_offset, _original_frame_duration, original_use_cyclic, ) = image_user_settings(source_object) source_image.use_fake_user = True source_object.data = result_image configure_image_user( image_user, len(foreground_paths), frame_start=original_frame_start, use_cyclic=original_use_cyclic, ) except (OSError, RuntimeError) as error: source_image.use_fake_user = original_fake_user bpy.data.images.remove(result_image) raise RuntimeError("Unable to load the background result") from error return result_image if not output_path.is_file(): raise RuntimeError( f"Background removal result does not exist: {output_path}" ) try: result_image = bpy.data.images.load( str(output_path), check_existing=False, ) except RuntimeError as error: raise RuntimeError("Unable to load the background result") from error source_image = source_object.data original_fake_user = source_image.use_fake_user try: result_image.name = removed_background_data_name(source_object) result_image.pack() if assign_to_source: source_image.use_fake_user = True source_object.data = result_image except (OSError, RuntimeError) as error: source_image.use_fake_user = original_fake_user bpy.data.images.remove(result_image) raise RuntimeError("Unable to pack the background result") from error return result_image def plane_bounds(source_object): image_width, image_height = source_object.data.size if image_width <= 0 or image_height <= 0: raise ValueError("The source image has no readable dimensions") aspect = image_width / image_height display_size = float(source_object.empty_display_size) if aspect >= 1.0: width = display_size height = display_size / aspect else: width = display_size * aspect height = display_size offset = source_object.empty_image_offset x_min = float(offset[0]) * width y_min = float(offset[1]) * height return ( x_min, x_min + width, y_min, y_min + height, ) def plane_fit_transform(source_object, camera_bounds): target_bounds = plane_bounds(source_object) camera_width = camera_bounds[1] - camera_bounds[0] camera_height = camera_bounds[3] - camera_bounds[2] if camera_width <= 0.0 or camera_height <= 0.0: raise ValueError("The camera plane has invalid dimensions") target_width = target_bounds[1] - target_bounds[0] target_height = target_bounds[3] - target_bounds[2] scale_x = target_width / camera_width scale_y = target_height / camera_height scale_z = (scale_x * scale_y) ** 0.5 camera_center_x = (camera_bounds[0] + camera_bounds[1]) * 0.5 camera_center_y = (camera_bounds[2] + camera_bounds[3]) * 0.5 target_center_x = (target_bounds[0] + target_bounds[1]) * 0.5 target_center_y = (target_bounds[2] + target_bounds[3]) * 0.5 return ( scale_x, scale_y, scale_z, target_center_x - camera_center_x * scale_x, target_center_y - camera_center_y * scale_y, ) def apply_plane_transform(plane_object, source_object, camera_bounds): from mathutils import Matrix scale_x, scale_y, scale_z, offset_x, offset_y = plane_fit_transform( source_object, camera_bounds, ) fit_matrix = Matrix( ( (scale_x, 0.0, 0.0, offset_x), (0.0, scale_y, 0.0, offset_y), (0.0, 0.0, scale_z, 0.0), (0.0, 0.0, 0.0, 1.0), ) ) plane_object.matrix_world = source_object.matrix_world @ fit_matrix return abs(float(plane_object.scale.z)) def extract_depth_arrays(npz_path): try: import numpy as np except ImportError as error: raise RuntimeError("Blender Python does not provide NumPy") from error try: with np.load(npz_path, allow_pickle=False) as archive: depth_key = next( ( key for key in DEPTH_KEY_CANDIDATES if key in archive.files ), None, ) if depth_key is None: keys = ", ".join(archive.files) or "none" raise ValueError( f"Mini NPZ has no depth array; available keys: {keys}" ) depth = np.asarray(archive[depth_key], dtype=np.float32) except OSError as error: raise ValueError(f"Unable to read Mini NPZ: {npz_path}") from error if depth.ndim >= 3 and depth.shape[-1] == 1: depth = depth[..., 0] if depth.ndim == 2: depth = depth[None, ...] if depth.ndim != 3: raise ValueError( f"Unsupported depth array shape: {tuple(depth.shape)}" ) return [depth[index] for index in range(depth.shape[0])] def extract_depth_array(npz_path): return extract_depth_arrays(npz_path)[0] def extract_intrinsics_array(npz_path): try: import numpy as np except ImportError as error: raise RuntimeError("Blender Python does not provide NumPy") from error try: with np.load(npz_path, allow_pickle=False) as archive: intrinsics_key = next( ( key for key in INTRINSICS_KEY_CANDIDATES if key in archive.files ), None, ) if intrinsics_key is None: keys = ", ".join(archive.files) or "none" raise ValueError( "Mini NPZ has no camera intrinsics; " f"available keys: {keys}" ) intrinsics = np.asarray( archive[intrinsics_key], dtype=np.float32, ) except OSError as error: raise ValueError(f"Unable to read Mini NPZ: {npz_path}") from error if intrinsics.ndim == 3: intrinsics = intrinsics[0] if intrinsics.shape != (3, 3): raise ValueError( f"Unsupported intrinsics shape: {tuple(intrinsics.shape)}" ) return intrinsics def prepare_depth_values(depth): import numpy as np valid = np.isfinite(depth) & (depth > 0.0) if not valid.any(): raise ValueError("Depth array contains no positive finite values") values = depth.astype(np.float32, copy=True) values[~valid] = 0.0 return values def camera_plane_bounds( source_object, depth, intrinsics, displacement_strength, ): import numpy as np scale = abs(float(displacement_strength)) if scale <= 1e-8: raise ValueError("Displacement Strength must not be zero") valid = np.isfinite(depth) & (depth > 0.0) if not valid.any(): raise ValueError("Depth array contains no positive finite values") reference_depth = float(np.median(depth[valid])) reference_distance = reference_depth * scale height, width = depth.shape fx = float(intrinsics[0, 0]) fy = float(intrinsics[1, 1]) cx = float(intrinsics[0, 2]) cy = float(intrinsics[1, 2]) if fx <= 0.0 or fy <= 0.0: raise ValueError("Camera intrinsics contain an invalid focal length") source_width, source_height = source_object.data.size if source_width <= 0 or source_height <= 0: raise ValueError("The source image has no readable dimensions") source_aspect = source_width / source_height # Predicted fx/fy may differ even for square pixels. Preserve their projected # area while enforcing the source image aspect ratio to avoid image stretch. raw_width = width * reference_distance / fx raw_height = height * reference_distance / fy plane_area = raw_width * raw_height plane_width = float(np.sqrt(plane_area * source_aspect)) plane_height = float(np.sqrt(plane_area / source_aspect)) x_min = -(cx / width) * plane_width x_max = (1.0 - cx / width) * plane_width y_min = -(1.0 - cy / height) * plane_height y_max = (cy / height) * plane_height # Preserve an intentional Image Empty origin offset around the camera frame. offset = source_object.empty_image_offset x_offset = (float(offset[0]) + 0.5) * plane_width y_offset = (float(offset[1]) + 0.5) * plane_height return ( ( x_min + x_offset, x_max + x_offset, y_min + y_offset, y_max + y_offset, ), reference_depth, ) def create_depth_image(name, depth_values): import numpy as np height, width = depth_values.shape image = bpy.data.images.new( name, width=width, height=height, alpha=False, float_buffer=True, ) image.colorspace_settings.name = "Non-Color" image.file_format = "OPEN_EXR" # NPZ arrays use a top-left origin, while Blender image pixels start below. depth_pixels = np.flipud(depth_values) rgba = np.empty((height, width, 4), dtype=np.float32) rgba[..., 0] = depth_pixels rgba[..., 1] = depth_pixels rgba[..., 2] = depth_pixels rgba[..., 3] = 1.0 image.pixels.foreach_set(rgba.ravel()) # Pack as EXR so values outside 0..1 survive saving and reopening the blend. image.pack() return image def grid_segments(bounds, subdivision_level): if subdivision_level < 0: raise ValueError("Subdivision must not be negative") width = bounds[1] - bounds[0] height = bounds[3] - bounds[2] if width <= 0.0 or height <= 0.0: raise ValueError("The plane has invalid dimensions") short_segments = 2**subdivision_level if width >= height: return ( max(short_segments, int(short_segments * width / height + 0.5)), short_segments, ) return ( short_segments, max(short_segments, int(short_segments * height / width + 0.5)), ) def create_plane_mesh(source_object, bounds=None, subdivision_level=5): x_min, x_max, y_min, y_max = ( bounds if bounds is not None else plane_bounds(source_object) ) x_segments, y_segments = grid_segments( (x_min, x_max, y_min, y_max), subdivision_level, ) vertices = [] uv_coordinates = [] for y_index in range(y_segments + 1): v = y_index / y_segments y = y_min + (y_max - y_min) * v for x_index in range(x_segments + 1): u = x_index / x_segments x = x_min + (x_max - x_min) * u vertices.append((x, y, 0.0)) uv_coordinates.append((u, v)) faces = [] row_width = x_segments + 1 for y_index in range(y_segments): for x_index in range(x_segments): bottom_left = y_index * row_width + x_index faces.append( ( bottom_left, bottom_left + 1, bottom_left + row_width + 1, bottom_left + row_width, ) ) mesh = bpy.data.meshes.new(source_object.name) mesh.from_pydata(vertices, (), faces) mesh.update() for polygon in mesh.polygons: polygon.use_smooth = True uv_layer = mesh.uv_layers.new(name="UVMap") for loop in mesh.loops: uv_layer.data[loop.index].uv = uv_coordinates[ loop.vertex_index ] return mesh def create_material( name, color_image, depth_scale, displacement_image=None, displacement_midlevel=0.0, displacement_scale=1.0, unshaded=False, frame_count=1, frame_start=1, color_frame_offset=0, color_frame_duration=None, depth_frame_offset=0, depth_frame_duration=None, use_cyclic=None, ): material = bpy.data.materials.new(name) material.use_nodes = True if hasattr(material, "surface_render_method"): material.surface_render_method = "DITHERED" nodes = material.node_tree.nodes nodes.clear() color_texture = nodes.new("ShaderNodeTexImage") output = nodes.new("ShaderNodeOutputMaterial") color_texture.image = color_image color_texture.interpolation = "Linear" color_texture.extension = "EXTEND" color_texture.location = (-640.0, 180.0) output.location = (460.0, 180.0) extracted_frames = frame_count if frame_count > 1 else None color_frames = ( color_frame_duration if color_frame_duration is not None else extracted_frames ) depth_frames = ( depth_frame_duration if depth_frame_duration is not None else extracted_frames ) if frame_count > 1 or is_animated_image(color_image): configure_image_user( color_texture.image_user, color_frames, frame_start=frame_start, frame_offset=color_frame_offset, use_cyclic=use_cyclic, ) links = material.node_tree.links if unshaded: emission = nodes.new("ShaderNodeEmission") transparent = nodes.new("ShaderNodeBsdfTransparent") mix_shader = nodes.new("ShaderNodeMixShader") emission.location = (-100.0, 260.0) transparent.location = (-100.0, 20.0) mix_shader.location = (200.0, 180.0) links.new( color_texture.outputs["Color"], emission.inputs["Color"], ) links.new( color_texture.outputs["Alpha"], mix_shader.inputs["Fac"], ) links.new( transparent.outputs["BSDF"], mix_shader.inputs[1], ) links.new( emission.outputs["Emission"], mix_shader.inputs[2], ) links.new( mix_shader.outputs["Shader"], output.inputs["Surface"], ) else: shader = nodes.new("ShaderNodeBsdfPrincipled") bump = nodes.new("ShaderNodeBump") roughness_curve = nodes.new("ShaderNodeFloatCurve") bump.inputs["Distance"].default_value = BUMP_BASE_DISTANCE * depth_scale roughness_curve.mapping.initialize() curve_points = roughness_curve.mapping.curves[0].points curve_points[0].location = (0.0, 0.5) curve_points[-1].location = (1.0, 0.1) roughness_curve.mapping.update() roughness_curve.location = (-300.0, 320.0) bump.location = (-300.0, -80.0) shader.location = (80.0, 180.0) links.new( color_texture.outputs["Color"], shader.inputs["Base Color"], ) links.new(color_texture.outputs["Alpha"], shader.inputs["Alpha"]) links.new(color_texture.outputs["Color"], bump.inputs["Height"]) links.new(bump.outputs["Normal"], shader.inputs["Normal"]) links.new( color_texture.outputs["Color"], roughness_curve.inputs["Value"], ) links.new( roughness_curve.outputs["Value"], shader.inputs["Roughness"], ) links.new(shader.outputs["BSDF"], output.inputs["Surface"]) if displacement_image is not None: depth_texture = nodes.new("ShaderNodeTexImage") displacement = nodes.new("ShaderNodeDisplacement") depth_texture.image = displacement_image depth_texture.interpolation = "Linear" depth_texture.extension = "EXTEND" if frame_count > 1 or is_animated_image(displacement_image): configure_image_user( depth_texture.image_user, depth_frames, frame_start=frame_start, frame_offset=depth_frame_offset, use_cyclic=use_cyclic, ) displacement.inputs["Midlevel"].default_value = ( displacement_midlevel ) displacement.inputs["Scale"].default_value = displacement_scale depth_texture.location = (-300.0, -360.0) displacement.location = (80.0, -260.0) links.new( depth_texture.outputs["Color"], displacement.inputs["Height"], ) links.new( displacement.outputs["Displacement"], output.inputs["Displacement"], ) set_displacement_only(material) return material def set_displacement_only(material): for owner, property_name in ( (material, "surface_displacement_method"), (material, "displacement_method"), (getattr(material, "cycles", None), "displacement_method"), ): if owner is not None and hasattr(owner, property_name): setattr(owner, property_name, "DISPLACEMENT") return def finish_depth_plane( context, source_object, plane_object, delete_source, ): source_name = source_object.name for selected in context.selected_objects: selected.select_set(False) plane_object.select_set(True) context.view_layer.objects.active = plane_object if delete_source: bpy.data.objects.remove(source_object, do_unlink=True) plane_object.name = source_name return plane_object def build_image_depth_plane( context, source_object, subdivision_level, delete_source, displacement_via="MODIFIER", unshaded=False, color_image=None, depth_frames_directory=None, ): source_name = source_object.name frame_count = 1 frames_image = None if depth_frames_directory is not None: frames_image = create_sequence_image( depth_data_name(source_object), depth_frames_directory, ) frame_count = len( sorted_frame_paths(depth_frames_directory, "frame.") ) ( source_frame_start, source_frame_offset, source_frame_duration, source_use_cyclic, ) = image_user_settings(source_object) uses_extracted = frames_image is not None texture_frame_count = ( frame_count if uses_extracted else source_frame_duration ) texture_frame_offset = 0 if uses_extracted else source_frame_offset depth_source_image = source_object.data if frames_image is not None: depth_source_image = frames_image elif not is_animated_image(source_object.data): depth_source_image = copy_depth_image(source_object) mesh = create_plane_mesh( source_object, subdivision_level=subdivision_level, ) plane_object = bpy.data.objects.new(source_name, mesh) context.collection.objects.link(plane_object) plane_object.matrix_world = source_object.matrix_world.copy() subdivision = plane_object.modifiers.new("Subdivision", "SUBSURF") subdivision.subdivision_type = "SIMPLE" subdivision.levels = 1 subdivision.render_levels = 2 if displacement_via == "MODIFIER": texture = bpy.data.textures.new( depth_data_name(source_object), type="IMAGE", ) texture.image = depth_source_image texture.extension = "EXTEND" if uses_extracted or is_animated_image(source_object.data): configure_image_user( texture.image_user, texture_frame_count, frame_start=source_frame_start, frame_offset=texture_frame_offset, use_cyclic=source_use_cyclic, ) displace = plane_object.modifiers.new("Displace", "DISPLACE") displace.texture = texture displace.texture_coords = "UV" displace.mid_level = ORIGINAL_DISPLACEMENT_MIDLEVEL depth_scale = abs(float(plane_object.scale.z)) displacement_image = ( depth_source_image if displacement_via == "MATERIAL" else None ) material = create_material( source_name, color_image if color_image is not None else source_object.data, depth_scale, displacement_image=displacement_image, displacement_midlevel=ORIGINAL_DISPLACEMENT_MIDLEVEL, displacement_scale=ORIGINAL_DISPLACEMENT_STRENGTH, unshaded=unshaded, frame_count=frame_count, frame_start=source_frame_start, color_frame_offset=texture_frame_offset, color_frame_duration=texture_frame_count, depth_frame_offset=texture_frame_offset, depth_frame_duration=texture_frame_count, use_cyclic=source_use_cyclic, ) mesh.materials.append(material) plane_object["snapform_frame_count"] = frame_count return finish_depth_plane( context, source_object, plane_object, delete_source, ) def build_depth_plane( context, source_object, npz_path, subdivision_level, delete_source, displacement_via="MODIFIER", unshaded=False, depth_name="", color_image=None, depth_frames_directory=None, ): source_name = source_object.name depth_arrays = extract_depth_arrays(npz_path) depth = depth_arrays[0] intrinsics = extract_intrinsics_array(npz_path) depth_values = prepare_depth_values(depth) bounds, reference_depth = camera_plane_bounds( source_object, depth_values, intrinsics, DISPLACEMENT_STRENGTH, ) depth_name = depth_name or depth_data_name(source_object) frame_count = len(depth_arrays) ( source_frame_start, source_frame_offset, source_frame_duration, source_use_cyclic, ) = image_user_settings(source_object) depth_image = create_depth_image( depth_name, depth_values, ) if frame_count > 1 and depth_frames_directory is not None: bpy.data.images.remove(depth_image, do_unlink=True) depth_image = create_depth_sequence_image( depth_name, [prepare_depth_values(frame) for frame in depth_arrays], context.scene, depth_frames_directory, ) mesh = create_plane_mesh(source_object, bounds, subdivision_level) plane_object = bpy.data.objects.new( source_name, mesh, ) context.collection.objects.link(plane_object) depth_scale = apply_plane_transform( plane_object, source_object, bounds, ) subdivision = plane_object.modifiers.new("Subdivision", "SUBSURF") subdivision.subdivision_type = "SIMPLE" subdivision.levels = 1 subdivision.render_levels = 2 if displacement_via == "MODIFIER": texture = bpy.data.textures.new( depth_name, type="IMAGE", ) texture.image = depth_image texture.extension = "EXTEND" if frame_count > 1: configure_image_user( texture.image_user, frame_count, frame_start=source_frame_start, use_cyclic=source_use_cyclic, ) displace = plane_object.modifiers.new("Displace", "DISPLACE") displace.texture = texture displace.texture_coords = "UV" displace.direction = "NORMAL" displace.strength = DISPLACEMENT_STRENGTH displace.mid_level = DISPLACEMENT_MIDLEVEL material = create_material( source_name, color_image if color_image is not None else source_object.data, depth_scale, displacement_image=( depth_image if displacement_via == "MATERIAL" else None ), displacement_midlevel=DISPLACEMENT_MIDLEVEL, displacement_scale=DISPLACEMENT_STRENGTH, unshaded=unshaded, frame_count=frame_count, frame_start=source_frame_start, color_frame_offset=0 if color_image is not None else source_frame_offset, color_frame_duration=( frame_count if color_image is not None else source_frame_duration ), depth_frame_offset=0, depth_frame_duration=frame_count, use_cyclic=source_use_cyclic, ) mesh.materials.append(material) plane_object["snapform_reference_depth"] = reference_depth plane_object["snapform_camera_plane_width"] = bounds[1] - bounds[0] plane_object["snapform_camera_plane_height"] = bounds[3] - bounds[2] plane_object["snapform_depth_scale"] = depth_scale plane_object["snapform_frame_count"] = frame_count return finish_depth_plane( context, source_object, plane_object, delete_source, ) class BuildDepthPlane(bpy.types.Operator): bl_idname = "snapform.build_depth_plane" bl_label = "Build Depth Plane" bl_options = {"INTERNAL"} snapform_source_object_name: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_npz_path: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_depth_data_name: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_background_path: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_color_frames_dir: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_depth_frames_dir: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_subdivision_level: bpy.props.IntProperty( default=5, min=0, max=10, options={"HIDDEN", "SKIP_SAVE"}, ) snapform_delete_source: bpy.props.BoolProperty( default=True, options={"HIDDEN", "SKIP_SAVE"}, ) snapform_displacement_via: bpy.props.EnumProperty( items=( ("MODIFIER", "Modifier", "Use a Displace modifier"), ("MATERIAL", "Material", "Use material displacement"), ), default="MODIFIER", options={"HIDDEN", "SKIP_SAVE"}, ) snapform_unshaded: bpy.props.BoolProperty( default=False, options={"HIDDEN", "SKIP_SAVE"}, ) def execute(self, context): source_object = bpy.data.objects.get(self.snapform_source_object_name) if not is_image_empty(source_object): self.report( {"ERROR"}, "The source image empty is no longer available", ) return {"CANCELLED"} npz_path = Path(self.snapform_npz_path) if not npz_path.is_file(): self.report( {"ERROR"}, f"Depth result does not exist: {npz_path}", ) return {"CANCELLED"} try: color_image = None if self.snapform_background_path: color_image = create_removed_background_image( source_object, self.snapform_background_path, assign_to_source=False, ) elif self.snapform_color_frames_dir: color_image = create_sequence_image( f"{image_base_name(source_object.data)}.frame", self.snapform_color_frames_dir, ) build_depth_plane( context, source_object, npz_path, self.snapform_subdivision_level, self.snapform_delete_source, displacement_via=self.snapform_displacement_via, unshaded=self.snapform_unshaded, depth_name=self.snapform_depth_data_name, color_image=color_image, depth_frames_directory=( self.snapform_depth_frames_dir or None ), ) except (OSError, RuntimeError, ValueError) as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} return {"FINISHED"} class BuildOriginalDepthPlane(bpy.types.Operator): bl_idname = "snapform.build_original_depth_plane" bl_label = "Build Original Depth Plane" bl_options = {"INTERNAL"} snapform_source_object_name: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_background_path: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_color_frames_dir: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_depth_frames_dir: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_subdivision_level: bpy.props.IntProperty( default=5, min=0, max=10, options={"HIDDEN", "SKIP_SAVE"}, ) snapform_delete_source: bpy.props.BoolProperty( default=True, options={"HIDDEN", "SKIP_SAVE"}, ) snapform_displacement_via: bpy.props.EnumProperty( items=( ("MODIFIER", "Modifier", "Use a Displace modifier"), ("MATERIAL", "Material", "Use material displacement"), ), default="MODIFIER", options={"HIDDEN", "SKIP_SAVE"}, ) snapform_unshaded: bpy.props.BoolProperty( default=False, options={"HIDDEN", "SKIP_SAVE"}, ) def execute(self, context): source_object = bpy.data.objects.get(self.snapform_source_object_name) if not is_image_empty(source_object): self.report( {"ERROR"}, "The source image empty is no longer available", ) return {"CANCELLED"} try: color_image = None if self.snapform_background_path: color_image = create_removed_background_image( source_object, self.snapform_background_path, assign_to_source=False, ) elif self.snapform_color_frames_dir: color_image = create_sequence_image( f"{image_base_name(source_object.data)}.frame", self.snapform_color_frames_dir, ) build_image_depth_plane( context, source_object, self.snapform_subdivision_level, self.snapform_delete_source, displacement_via=self.snapform_displacement_via, unshaded=self.snapform_unshaded, color_image=color_image, depth_frames_directory=( self.snapform_depth_frames_dir or None ), ) except (OSError, RuntimeError, ValueError) as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} return {"FINISHED"} class ApplyRemovedBackground(bpy.types.Operator): bl_idname = "snapform.apply_removed_background" bl_label = "Apply Removed Background" bl_options = {"INTERNAL"} snapform_source_object_name: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) snapform_output_path: bpy.props.StringProperty( options={"HIDDEN", "SKIP_SAVE"}, ) def execute(self, _context): source_object = bpy.data.objects.get(self.snapform_source_object_name) if not is_image_empty(source_object): self.report( {"ERROR"}, "The source image empty is no longer available", ) return {"CANCELLED"} try: create_removed_background_image( source_object, self.snapform_output_path, ) except RuntimeError as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} return {"FINISHED"} class RemoveImageBackground(bpy.types.Operator): bl_idname = "snapform.remove_image_background" bl_label = "Remove Background" bl_description = "Remove the background from this Image Empty with BEN2" bl_options = {"REGISTER"} @classmethod def poll(cls, context): from .operators import active_job from .runtime import host_busy return ( is_image_empty(context.object) and active_job() is None and not host_busy() ) def execute(self, context): from .operators import IMAGE_BACKGROUND_JOB source_object = context.object total_frames = video_total_frames(source_object.data) if total_frames > 1: context.scene.snapform_settings.video_frames = ( default_extract_frames(source_object) ) try: source_path, delete_input = prepare_image_input( source_object.data, context.scene, ) except RuntimeError as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} result = bpy.ops.snapform.run_job( job_type=IMAGE_BACKGROUND_JOB, snapform_input_path=str(source_path), snapform_source_object_name=source_object.name, snapform_delete_input=delete_input, ) if "CANCELLED" not in result: return {"FINISHED"} if delete_input: remove_temporary_input(source_path) return {"CANCELLED"} class ConvertImageToDepthPlane(bpy.types.Operator): bl_idname = "snapform.convert_image_to_depth_plane" bl_label = "Convert to Depth Plane" bl_description = "Convert an image to a displaced depth plane" bl_options = {"REGISTER"} snapform_depth_source: bpy.props.EnumProperty( name="Depth from", description="Source used to displace the plane", items=( ( "ESTIMATE", "AI Generated", "Generate depth with an AI model", ), ("ORIGINAL", "Original", "Use the original image as depth"), ), default="ORIGINAL", ) snapform_displacement_via: bpy.props.EnumProperty( name="Displace via", description="How the depth displacement is presented", items=( ("MODIFIER", "Modifier", "Use a Displace modifier"), ("MATERIAL", "Material", "Use material displacement"), ), default="MODIFIER", ) snapform_unshaded: bpy.props.BoolProperty( name="Unshaded", description="Display the source color through an Emission shader", default=False, ) snapform_remove_background: bpy.props.BoolProperty( name="Remove Background", description="Remove the color image background with BEN2", default=False, ) snapform_model: bpy.props.EnumProperty( name="Model", description="Model used to estimate image depth", items=image_depth_model_items, ) snapform_use_ray_pose: bpy.props.BoolProperty( name="Use Ray Pose", description="Use ray-based camera pose estimation", default=True, ) snapform_process_res_factor: bpy.props.FloatProperty( name="Resolution Scale", description="Scale applied to the source image long-edge resolution", default=1.0, min=0.1, max=1.0, step=10, precision=2, ) snapform_delete_original: bpy.props.BoolProperty( name="Delete Original", description="Delete the source Image Empty after conversion", default=True, ) snapform_subdivision_level: bpy.props.IntProperty( name="Subdivision", description="Power of two for the short-edge segment count; the long edge follows the image aspect ratio", default=5, min=0, max=10, ) @classmethod def poll(cls, context): from .operators import active_job from .runtime import host_busy return ( is_image_empty(context.object) and active_job() is None and not host_busy() ) def invoke(self, context, _event): global _last_depth_plane_model if not self.properties.is_property_set("snapform_model"): available_keys = [ item[0] for item in image_depth_model_items(self, context) ] if available_keys: self.snapform_model = ( _last_depth_plane_model if _last_depth_plane_model in available_keys else available_keys[0] ) return context.window_manager.invoke_props_dialog(self, width=340) def draw(self, context): layout = self.layout layout.prop(self, "snapform_depth_source") if self.snapform_depth_source == "ESTIMATE": estimate_box = layout.box() available_models = image_depth_model_items(self, context) if not available_models: estimate_box.label(text="No DA3 models installed", icon="ERROR") estimate_box.label(text="Download one in Host > Models") else: estimate_box.prop(self, "snapform_model") estimate_box.prop(self, "snapform_use_ray_pose") process_res = scaled_process_resolution( context.object.data, self.snapform_process_res_factor, ) estimate_box.prop( self, "snapform_process_res_factor", text=f"Resolution Scale ({process_res} px)", ) settings_header, settings_body = layout.panel( "snapform_depth_plane_options", default_closed=True, ) settings_header.label(text="Options") if settings_body is not None: settings_body.prop(self, "snapform_displacement_via") settings_body.prop(self, "snapform_subdivision_level") settings_body.prop(self, "snapform_remove_background") settings_body.prop(self, "snapform_unshaded") layout.prop(self, "snapform_delete_original") def execute(self, context): from .operators import ( IMAGE_DEPTH_JOB, ORIGINAL_BACKGROUND_JOB, ORIGINAL_FRAMES_JOB, ) source_object = context.object if video_total_frames(source_object.data) > 1: context.scene.snapform_settings.video_frames = ( default_extract_frames(source_object) ) if ( self.snapform_depth_source == "ORIGINAL" and not self.snapform_remove_background ): if video_total_frames(source_object.data) > 1: try: source_path, delete_input = prepare_image_input( source_object.data, context.scene, ) except RuntimeError as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} result = bpy.ops.snapform.run_job( job_type=ORIGINAL_FRAMES_JOB, snapform_input_path=str(source_path), snapform_source_object_name=source_object.name, snapform_subdivision_level=self.snapform_subdivision_level, snapform_delete_source=self.snapform_delete_original, snapform_delete_input=delete_input, snapform_displacement_via=self.snapform_displacement_via, snapform_unshaded=self.snapform_unshaded, ) if "CANCELLED" not in result: return {"FINISHED"} if delete_input: remove_temporary_input(source_path) return {"CANCELLED"} try: build_image_depth_plane( context, source_object, self.snapform_subdivision_level, self.snapform_delete_original, self.snapform_displacement_via, self.snapform_unshaded, ) except (OSError, RuntimeError, ValueError) as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} undo_result = bpy.ops.ed.undo_push( message="Convert to Depth Plane", ) if "FINISHED" not in undo_result: self.report( {"ERROR"}, "Unable to create the depth plane Undo step", ) return {"CANCELLED"} return {"FINISHED"} if self.snapform_depth_source == "ORIGINAL": try: source_path, delete_input = prepare_image_input( source_object.data, context.scene, ) except RuntimeError as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} result = bpy.ops.snapform.run_job( job_type=ORIGINAL_BACKGROUND_JOB, snapform_input_path=str(source_path), snapform_source_object_name=source_object.name, snapform_subdivision_level=self.snapform_subdivision_level, snapform_delete_source=self.snapform_delete_original, snapform_delete_input=delete_input, snapform_displacement_via=self.snapform_displacement_via, snapform_unshaded=self.snapform_unshaded, ) if "CANCELLED" not in result: return {"FINISHED"} if delete_input: remove_temporary_input(source_path) return {"CANCELLED"} image = source_object.data try: process_res = scaled_process_resolution( image, self.snapform_process_res_factor, ) except ValueError as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} try: source_path, delete_input = prepare_image_input( image, context.scene, ) except RuntimeError as error: self.report({"ERROR"}, str(error)) return {"CANCELLED"} result = bpy.ops.snapform.run_job( job_type=IMAGE_DEPTH_JOB, snapform_input_path=str(source_path), snapform_source_object_name=source_object.name, snapform_depth_data_name=depth_data_name(source_object), snapform_model_key=self.snapform_model, snapform_use_ray_pose=self.snapform_use_ray_pose, snapform_process_res=process_res, snapform_subdivision_level=self.snapform_subdivision_level, snapform_delete_source=self.snapform_delete_original, snapform_delete_input=delete_input, snapform_displacement_via=self.snapform_displacement_via, snapform_unshaded=self.snapform_unshaded, snapform_remove_background=self.snapform_remove_background, ) if "CANCELLED" in result: if delete_input: remove_temporary_input(source_path) return {"CANCELLED"} global _last_depth_plane_model _last_depth_plane_model = self.snapform_model return {"FINISHED"} def draw_image_context_menu(self, context): active_object = context.object if not is_image_empty(active_object): return self.layout.separator() row = self.layout.row() row.operator_context = "INVOKE_DEFAULT" row.operator( "snapform.convert_image_to_depth_plane", text="Convert to Depth Plane", icon="MOD_DISPLACE", ) row = self.layout.row() row.operator_context = "INVOKE_DEFAULT" row.operator( "snapform.remove_image_background", text="Remove Background", icon="IMAGE_ALPHA", )