Add image perspective editing and related tools

- Implemented the Image Perspective tool for editing image corners in Blender.
- Created a new operator `EditImagePerspective` to handle the perspective transformation.
- Added a preview feature for the perspective adjustment.
- Introduced the `ImagePolylineTool` for editing image masks with polylines.
- Developed backend support for Real-ESRGAN model inference, including ONNX integration.
- Added functionality for image masking and background removal.
- Created unit tests for the Real-ESRGAN ONNX model to ensure correct preprocessing and inference.

src/anyimage/server/__init__.py

@@ -9,8 +9,9 @@ from .jobs.balloon import run as _analyze_balloon
 from .jobs.debug_ben2 import run as _debug_ben2
 from .jobs.debug_da3 import run as _debug_da3
 from .jobs.debug_moge2 import run as _debug_moge2
+from .jobs.debug_realesrgan import run as _debug_realesrgan
 from .jobs.frames import run as _extract_frames
-from .jobs.image_extract import run as _extract_image_background
+from .jobs.image_mask import run as _refine_image_mask
 from .model_manager import ModelManager, ModelSessionCache
 
 
@@ -39,9 +40,9 @@ def analyze_balloon(context, parameters):
     return _analyze_balloon(context, parameters)
 
 
-@server.job("extract-image-background")
-def extract_image_background(context, parameters):
-    return _extract_image_background(context, parameters)
+@server.job("refine-image-mask")
+def refine_image_mask(context, parameters):
+    return _refine_image_mask(context, parameters)
 
 
 @server.job("remove-background")
@@ -79,4 +80,11 @@ def debug_ben2(context, parameters):
     return _debug_ben2(context, parameters)
 
 
+@server.job("debug-realesrgan-general-x4v3")
+def debug_realesrgan(context, parameters):
+    return _debug_realesrgan(context, parameters)
+
+
+
+
 __all__ = ("ModelManager", "ModelSessionCache", "server")

src/anyimage/server/jobs/debug_realesrgan.py

@@ -0,0 +1,26 @@
+from ..media import copy_frames
+from ..models.realesrgan import infer_one
+from .common import input_frames
+
+
+def run(context, parameters):
+    context.progress(0.02, "Preparing Real-ESRGAN API input")
+    context.directory.mkdir(parents=True, exist_ok=True)
+    with input_frames(context, parameters) as images:
+        for index, image in enumerate(images):
+            context.check_cancelled()
+            prediction = infer_one(
+                context.resource("model_manager"),
+                parameters,
+                image,
+            )
+            suffix = "" if len(images) == 1 else f".{index + 1:04d}"
+            prediction.save(
+                context.directory / f"prediction{suffix}.png",
+                format="PNG",
+                compress_level=1,
+            )
+        copy_frames(images, context.directory)
+    context.check_cancelled()
+    context.progress(1.0, "Saved raw Real-ESRGAN API prediction")
+    return {"directory": "."}

src/anyimage/server/jobs/image_mask.py

@@ -67,13 +67,11 @@ def apply_selection_mask(foreground, selection_mask):
     return Image.fromarray(pixels, mode="RGBA")
 
 
-def crop_to_alpha(image, bounds, padding=0):
-    """Crop an RGBA image to nonzero Alpha and expand it by pixel padding."""
+def crop_to_alpha(image, bounds):
+    """Crop an RGBA mask image to its nonzero Alpha bounds."""
     import numpy as np
     from PIL import Image
 
-    if padding < 0:
-        raise ValueError("Padding must not be negative")
     pixels = np.asarray(image.convert("RGBA"), dtype=np.uint8)
     visible_y, visible_x = np.nonzero(pixels[:, :, 3] > 0)
     if not len(visible_x):
@@ -85,23 +83,23 @@ def crop_to_alpha(image, bounds, padding=0):
     output = Image.new(
         "RGBA",
         (
-            content_right - content_left + padding * 2,
-            content_bottom - content_top + padding * 2,
+            content_right - content_left,
+            content_bottom - content_top,
         ),
         (0, 0, 0, 0),
     )
-    output.paste(image, (padding - content_left, padding - content_top))
+    output.paste(image, (-content_left, -content_top))
     left, top, _right, _bottom = bounds
     return output, (
-        left + content_left - padding,
-        top + content_top - padding,
-        left + content_right + padding,
-        top + content_bottom + padding,
+        left + content_left,
+        top + content_top,
+        left + content_right,
+        top + content_bottom,
     )
 
 
 def run(context, parameters):
-    """Remove the background from an image selection and return source bounds."""
+    """Refine a gesture mask with BEN2 and return its source-image bounds."""
     started = time.perf_counter()
     context.progress(0.05, "Preparing image selection")
     context.check_cancelled()
@@ -124,15 +122,13 @@ def run(context, parameters):
     context.check_cancelled()
     context.progress(0.9, "Applying selection mask")
     output, bounds = crop_to_alpha(
-        apply_selection_mask(foreground, selection_mask),
-        bounds,
-        int(parameters.get("padding", 0)),
+        apply_selection_mask(foreground, selection_mask), bounds
     )
     output_path = output_directory / "image.png"
     output.save(output_path, format="PNG", compress_level=1)
     bounds_path = output_directory / "bounds.json"
     bounds_path.write_text(json.dumps(bounds, separators=(",", ":")), encoding="utf-8")
     elapsed_ms = (time.perf_counter() - started) * 1000.0
-    message = f"Image background removal finished in {elapsed_ms:.0f} ms"
+    message = f"Image selection refined in {elapsed_ms:.0f} ms"
     context.progress(1.0, message)
     return {"image": output_path.name, "bounds": bounds_path.name}

src/anyimage/server/model_catalog.py

@@ -89,6 +89,30 @@ BEN2_MODEL = ModelSpec(
 )
 
 
+REALESRGAN_MODEL = ModelSpec(
+    key="REALESRGAN_GENERAL_X4V3",
+    label="Real-ESRGAN General x4v3",
+    directory_name="realesr-general-x4v3",
+    repository="jonathanst29/tinier-upscale-models",
+    license_name="BSD 3-Clause",
+    description="Fast general-purpose 4x image upscaler using ONNX Runtime",
+    supports_gs=False,
+    download_patterns=("realesr-general-x4v3.onnx",),
+    ready_patterns=("realesr-general-x4v3.onnx",),
+    r2_directory="realesr-general-x4v3",
+    r2_files=((
+        "realesr-general-x4v3.onnx",
+        4866396,
+        "924ebad6532777303582d4ce7811849b88869231a3ae7093e0f21200249df8d5",
+    ),),
+)
+
+
+UPSCALE_MODELS = {
+    REALESRGAN_MODEL.key: REALESRGAN_MODEL,
+}
+
+
 MOGE2_MODELS = {
     "MOGE2_VITS_NORMAL": ModelSpec(
         key="MOGE2_VITS_NORMAL",
@@ -146,6 +170,7 @@ MOGE2_MODEL = MOGE2_MODELS["MOGE2_VITS_NORMAL"]
 DOWNLOADABLE_MODELS = {
     **MODELS,
     BEN2_MODEL.key: BEN2_MODEL,
+    **UPSCALE_MODELS,
     **MOGE2_MODELS,
 }
 MODEL_NUMBERS = {
@@ -181,8 +206,14 @@ def model_record(model, ready=False):
         "license": model.license_name,
         "supports_gs": model.supports_gs,
         "enabled": model.enabled,
-        "family": "moge2" if model.key.startswith("MOGE2_") else (
-            "ben2" if model.key == BEN2_MODEL.key else "da3"
+        "family": (
+            "moge2"
+            if model.key.startswith("MOGE2_")
+            else "ben2"
+            if model.key == BEN2_MODEL.key
+            else "upscale"
+            if model.key in UPSCALE_MODELS
+            else "da3"
         ),
         "ready": bool(ready),
     }

src/anyimage/server/model_manager.py

@@ -17,6 +17,8 @@ class ModelSessionCache:
         self.da3_key = None
         self.moge_model = None
         self.moge_key = None
+        self.realesrgan_model = None
+        self.realesrgan_key = None
 
     def get_ben2(self, model_dir, device):
         requested_device = getattr(device, "type", device)
@@ -68,6 +70,18 @@ class ModelSessionCache:
         self.moge_key = key
         return model
 
+    def get_realesrgan(self, model_dir, requested_device):
+        from .models.onnx_realesrgan import create_session
+
+        key = (str(Path(model_dir).resolve()), requested_device, "onnx")
+        if self.realesrgan_model is not None and self.realesrgan_key == key:
+            return self.realesrgan_model
+        self.release_realesrgan()
+        model = create_session(model_dir, requested_device)
+        self.realesrgan_model = model
+        self.realesrgan_key = key
+        return model
+
     def release_ben2(self):
         self.ben2_model = None
         self.ben2_key = None
@@ -83,13 +97,20 @@ class ModelSessionCache:
         self.moge_key = None
         self._release_runtime_cache()
 
+    def release_realesrgan(self):
+        self.realesrgan_model = None
+        self.realesrgan_key = None
+        self._release_runtime_cache()
+
     def close(self):
         self.ben2_model = None
         self.da3_model = None
         self.moge_model = None
+        self.realesrgan_model = None
         self.ben2_key = None
         self.da3_key = None
         self.moge_key = None
+        self.realesrgan_key = None
         self._release_runtime_cache()
 
     def snapshot(self):
@@ -97,6 +118,11 @@ class ModelSessionCache:
             "ben2": Path(self.ben2_key[0]).name if self.ben2_key else "",
             "da3": Path(self.da3_key[0]).name if self.da3_key else "",
             "moge2": Path(self.moge_key[0]).name if self.moge_key else "",
+            "realesrgan": (
+                Path(self.realesrgan_key[0]).name
+                if self.realesrgan_key
+                else ""
+            ),
         }
 
     def _release_runtime_cache(self):
@@ -143,6 +169,9 @@ class ModelManager:
     def get_moge2(self, model_dir, device):
         return self.sessions.get_moge2(model_dir, device)
 
+    def get_realesrgan(self, model_dir, device):
+        return self.sessions.get_realesrgan(model_dir, device)
+
     def snapshot(self):
         return {"loaded": self.sessions.snapshot()}
 

src/anyimage/server/models/__init__.py

@@ -1,8 +1,14 @@
-from . import ben2, da3, moge2
+from . import ben2, da3, moge2, realesrgan
 
 
 def geometry_adapter(model_key):
     return moge2 if model_key.startswith("MOGE2_") else da3
 
 
-__all__ = ("ben2", "da3", "moge2", "geometry_adapter")
+__all__ = (
+    "ben2",
+    "da3",
+    "moge2",
+    "realesrgan",
+    "geometry_adapter",
+)

src/anyimage/server/models/ben2.py

@@ -32,15 +32,12 @@ def infer_foreground(
     check_cancelled(cancel_check)
     with Image.open(input_path) as opened_image:
         source_rgba = opened_image.convert("RGBA")
-    if preserve_input_alpha:
-        image = Image.new("RGB", source_rgba.size, (0, 0, 0))
-        image.paste(
-            source_rgba.convert("RGB"),
-            (0, 0),
-            source_rgba.getchannel("A"),
-        )
-    else:
-        image = source_rgba.convert("RGB")
+    image = Image.new("RGB", source_rgba.size, (0, 0, 0))
+    image.paste(
+        source_rgba.convert("RGB"),
+        (0, 0),
+        source_rgba.getchannel("A"),
+    )
     inference_started = time.perf_counter()
     foreground = infer(model, image)
     inference_ms = (time.perf_counter() - inference_started) * 1000.0

src/anyimage/server/models/onnx_realesrgan.py

@@ -0,0 +1,51 @@
+"""Real-ESRGAN General x4v3 ONNX inference."""
+
+from pathlib import Path
+
+try:
+    from .onnx_runtime import create_session as create_runtime_session
+except ImportError:
+    from onnx_runtime import create_session as create_runtime_session
+
+
+ONNX_FILENAME = "realesr-general-x4v3.onnx"
+SCALE = 4
+
+
+def create_session(model_directory, requested_device="auto"):
+    path = Path(model_directory) / ONNX_FILENAME
+    return create_runtime_session(path, requested_device)
+
+
+def preprocess(image):
+    import numpy as np
+
+    image_array = np.asarray(image.convert("RGB"), dtype=np.float32) / 255.0
+    return np.ascontiguousarray(image_array.transpose(2, 0, 1)[None])
+
+
+def infer(session, image):
+    import numpy as np
+    from PIL import Image
+
+    input_tensor = preprocess(image)
+    input_name = session.get_inputs()[0].name
+    output_name = session.get_outputs()[0].name
+    prediction = np.asarray(
+        session.run([output_name], {input_name: input_tensor})[0],
+        dtype=np.float32,
+    )
+    expected_shape = (
+        1,
+        3,
+        input_tensor.shape[2] * SCALE,
+        input_tensor.shape[3] * SCALE,
+    )
+    if prediction.shape != expected_shape:
+        raise RuntimeError(
+            "Real-ESRGAN General x4v3 returned an unexpected output shape: "
+            f"{prediction.shape}"
+        )
+    pixels = np.clip(prediction[0].transpose(1, 2, 0), 0.0, 1.0)
+    pixels = np.rint(pixels * 255.0).astype(np.uint8)
+    return Image.fromarray(pixels, mode="RGB")

src/anyimage/server/models/realesrgan.py

@@ -0,0 +1,12 @@
+def infer_one(model_manager, parameters, image_path):
+    from PIL import Image
+
+    from .onnx_realesrgan import infer
+
+    model_directory = model_manager.directory(parameters["model"])
+    model = model_manager.get_realesrgan(
+        model_directory,
+        parameters["device"],
+    )
+    with Image.open(image_path) as opened:
+        return infer(model, opened)