Add tools and tests for AnyImage model management and documentation
- Implement sync_models.py for preparing and uploading ONNX artifacts. - Create build_node.py for building and validating Blender scripts. - Add docs.py for building and previewing documentation with Mike or MkDocs. - Introduce model_exports.py for exporting PyTorch models to ONNX format. - Develop pack.py for packaging the project with dependencies for different platforms. - Add test_sync_models.py to validate sync_models functionality. - Add test_tool_commands.py to ensure repository tools have correct command scripts. - Initialize tools package with __init__.py.
This commit is contained in:
1
tools/__init__.py
Normal file
1
tools/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""AnyImage repository maintenance commands."""
|
||||
91
tools/build_node.py
Normal file
91
tools/build_node.py
Normal file
@@ -0,0 +1,91 @@
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
BLENDER_SCRIPTS = (
|
||||
"build.py",
|
||||
"validate.py",
|
||||
"validate_mesh_plane.py",
|
||||
"validate_cutout.py",
|
||||
)
|
||||
|
||||
|
||||
def project_env_value(name, path=None):
|
||||
env_path = path or PROJECT_ROOT / ".env"
|
||||
if not env_path.is_file():
|
||||
return None
|
||||
for raw_line in env_path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if line.startswith("export "):
|
||||
line = line.removeprefix("export ").lstrip()
|
||||
key, separator, value = line.partition("=")
|
||||
if separator and key.strip() == name:
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
||||
value = value[1:-1]
|
||||
return value or None
|
||||
return None
|
||||
|
||||
|
||||
def blender_path(explicit=None):
|
||||
candidate = (
|
||||
explicit
|
||||
or os.environ.get("BLENDER_BIN")
|
||||
or project_env_value("BLENDER_BIN")
|
||||
or shutil.which("blender")
|
||||
)
|
||||
if not candidate:
|
||||
raise RuntimeError(
|
||||
"Blender not found: pass --blender, set BLENDER_BIN, add it to .env, "
|
||||
"or add blender to PATH"
|
||||
)
|
||||
path = Path(candidate).expanduser()
|
||||
if not path.is_file():
|
||||
raise RuntimeError(f"Blender executable does not exist: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def run(command):
|
||||
print("+", subprocess.list2cmdline([str(part) for part in command]), flush=True)
|
||||
subprocess.run(command, cwd=PROJECT_ROOT, check=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build and validate O_AnyImage.blend.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--blender",
|
||||
help="Blender executable; overrides BLENDER_BIN, .env, and PATH.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-tests",
|
||||
action="store_true",
|
||||
help="Skip the Python test suite.",
|
||||
)
|
||||
options = parser.parse_args()
|
||||
executable = blender_path(options.blender)
|
||||
|
||||
for script_name in BLENDER_SCRIPTS:
|
||||
run(
|
||||
(
|
||||
executable,
|
||||
"--factory-startup",
|
||||
"--background",
|
||||
"--python",
|
||||
PROJECT_ROOT / "tools" / "node_assets" / script_name,
|
||||
)
|
||||
)
|
||||
if not options.skip_tests:
|
||||
run((sys.executable, "-m", "pytest"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
48
tools/docs.py
Normal file
48
tools/docs.py
Normal file
@@ -0,0 +1,48 @@
|
||||
import argparse
|
||||
import subprocess
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def project_version():
|
||||
"""Return the documentation version declared by the project."""
|
||||
project = tomllib.loads(
|
||||
(PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8")
|
||||
)["project"]
|
||||
return project["version"]
|
||||
|
||||
|
||||
def documentation_command(dev=False):
|
||||
"""Return the Mike or MkDocs command for the requested mode."""
|
||||
if dev:
|
||||
return ["mkdocs", "serve"]
|
||||
return [
|
||||
"mike",
|
||||
"deploy",
|
||||
"--update-aliases",
|
||||
project_version(),
|
||||
"latest",
|
||||
]
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description="Build or preview documentation.")
|
||||
parser.add_argument(
|
||||
"-dev",
|
||||
"--dev",
|
||||
action="store_true",
|
||||
help="preview the current documentation with live reload",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
subprocess.run(
|
||||
documentation_command(args.dev),
|
||||
cwd=PROJECT_ROOT,
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
204
tools/model_exports.py
Normal file
204
tools/model_exports.py
Normal file
@@ -0,0 +1,204 @@
|
||||
"""Export official PyTorch super-resolution models to ONNX."""
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
OPSET_VERSION = 17
|
||||
|
||||
|
||||
def _require_build_dependencies():
|
||||
try:
|
||||
import onnx
|
||||
import onnxruntime
|
||||
import torch
|
||||
except ImportError as error:
|
||||
raise RuntimeError(
|
||||
"Model build dependencies are missing. Run with: "
|
||||
"uv sync"
|
||||
) from error
|
||||
return onnx, onnxruntime, torch
|
||||
|
||||
|
||||
def _load_weights(torch, checkpoint_path, key):
|
||||
checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
|
||||
return checkpoint[key]
|
||||
|
||||
|
||||
def _validate_export(onnx, onnxruntime, torch, model, sample, destination):
|
||||
onnx.checker.check_model(onnx.load(destination))
|
||||
expected = model(sample).detach().numpy()
|
||||
session = onnxruntime.InferenceSession(
|
||||
str(destination),
|
||||
providers=["CPUExecutionProvider"],
|
||||
)
|
||||
actual = session.run(None, {"input": sample.numpy()})[0]
|
||||
np.testing.assert_allclose(actual, expected, rtol=1e-4, atol=1e-5)
|
||||
|
||||
|
||||
def _export(torch, model, sample, destination, dynamic_axes=None):
|
||||
model.eval()
|
||||
torch.onnx.export(
|
||||
model,
|
||||
sample,
|
||||
destination,
|
||||
export_params=True,
|
||||
opset_version=OPSET_VERSION,
|
||||
do_constant_folding=True,
|
||||
input_names=["input"],
|
||||
output_names=["output"],
|
||||
dynamic_axes=dynamic_axes,
|
||||
dynamo=False,
|
||||
)
|
||||
|
||||
|
||||
def _restore_realesrgan_scope_names(onnx, destination):
|
||||
model = onnx.load(destination)
|
||||
for node in model.graph.node:
|
||||
node.name = node.name.replace("/body.", "/body/body.")
|
||||
for values in (node.input, node.output):
|
||||
for index, value in enumerate(values):
|
||||
values[index] = value.replace("/body.", "/body/body.")
|
||||
onnx.save(model, destination)
|
||||
|
||||
|
||||
def export_realesrgan_wdn_x4(checkpoint_path, destination):
|
||||
onnx, onnxruntime, torch = _require_build_dependencies()
|
||||
nn = torch.nn
|
||||
|
||||
class SRVGGNetCompact(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.upscale = 4
|
||||
self.body = nn.ModuleList([nn.Conv2d(3, 64, 3, 1, 1), nn.PReLU(64)])
|
||||
for _ in range(32):
|
||||
self.body.extend([nn.Conv2d(64, 64, 3, 1, 1), nn.PReLU(64)])
|
||||
self.body.append(nn.Conv2d(64, 3 * self.upscale**2, 3, 1, 1))
|
||||
self.upsampler = nn.PixelShuffle(self.upscale)
|
||||
|
||||
def forward(self, value):
|
||||
result = value
|
||||
for layer in self.body:
|
||||
result = layer(result)
|
||||
result = self.upsampler(result)
|
||||
return result + torch.nn.functional.interpolate(
|
||||
value,
|
||||
scale_factor=self.upscale,
|
||||
mode="nearest",
|
||||
)
|
||||
|
||||
model = SRVGGNetCompact()
|
||||
model.load_state_dict(_load_weights(torch, checkpoint_path, "params"), strict=True)
|
||||
sample = torch.rand(1, 3, 16, 16)
|
||||
dynamic_axes = {
|
||||
"input": {2: "height", 3: "width"},
|
||||
"output": {2: "height4", 3: "width4"},
|
||||
}
|
||||
_export(torch, model, sample, destination, dynamic_axes)
|
||||
_restore_realesrgan_scope_names(onnx, destination)
|
||||
_validate_export(onnx, onnxruntime, torch, model, sample, destination)
|
||||
|
||||
|
||||
def export_safmn_real_x4(checkpoint_path, destination):
|
||||
onnx, onnxruntime, torch = _require_build_dependencies()
|
||||
nn = torch.nn
|
||||
functional = torch.nn.functional
|
||||
|
||||
class LayerNorm(nn.Module):
|
||||
def __init__(self, channels):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(channels))
|
||||
self.bias = nn.Parameter(torch.zeros(channels))
|
||||
self.eps = 1e-6
|
||||
|
||||
def forward(self, value):
|
||||
mean = value.mean(1, keepdim=True)
|
||||
variance = (value - mean).pow(2).mean(1, keepdim=True)
|
||||
value = (value - mean) / torch.sqrt(variance + self.eps)
|
||||
return self.weight[:, None, None] * value + self.bias[:, None, None]
|
||||
|
||||
class SAFM(nn.Module):
|
||||
def __init__(self, channels):
|
||||
super().__init__()
|
||||
chunk_channels = channels // 4
|
||||
self.mfr = nn.ModuleList(
|
||||
nn.Conv2d(chunk_channels, chunk_channels, 3, 1, 1, groups=chunk_channels)
|
||||
for _ in range(4)
|
||||
)
|
||||
self.aggr = nn.Conv2d(channels, channels, 1)
|
||||
self.act = nn.GELU()
|
||||
|
||||
def forward(self, value):
|
||||
height, width = value.shape[-2:]
|
||||
outputs = []
|
||||
for level, chunk in enumerate(value.chunk(4, dim=1)):
|
||||
if level:
|
||||
size = (height // 2**level, width // 2**level)
|
||||
chunk = functional.adaptive_max_pool2d(chunk, size)
|
||||
chunk = self.mfr[level](chunk)
|
||||
if level:
|
||||
chunk = functional.interpolate(
|
||||
chunk,
|
||||
size=(height, width),
|
||||
mode="nearest",
|
||||
)
|
||||
outputs.append(chunk)
|
||||
return self.act(self.aggr(torch.cat(outputs, dim=1))) * value
|
||||
|
||||
class CCM(nn.Module):
|
||||
def __init__(self, channels):
|
||||
super().__init__()
|
||||
self.ccm = nn.Sequential(
|
||||
nn.Conv2d(channels, channels * 2, 3, 1, 1),
|
||||
nn.GELU(),
|
||||
nn.Conv2d(channels * 2, channels, 1),
|
||||
)
|
||||
|
||||
def forward(self, value):
|
||||
return self.ccm(value)
|
||||
|
||||
class AttBlock(nn.Module):
|
||||
def __init__(self, channels):
|
||||
super().__init__()
|
||||
self.norm1 = LayerNorm(channels)
|
||||
self.norm2 = LayerNorm(channels)
|
||||
self.safm = SAFM(channels)
|
||||
self.ccm = CCM(channels)
|
||||
|
||||
def forward(self, value):
|
||||
value = self.safm(self.norm1(value)) + value
|
||||
return self.ccm(self.norm2(value)) + value
|
||||
|
||||
class SAFMN(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.to_feat = nn.Conv2d(3, 128, 3, 1, 1)
|
||||
self.feats = nn.Sequential(*(AttBlock(128) for _ in range(16)))
|
||||
self.to_img = nn.Sequential(
|
||||
nn.Conv2d(128, 3 * 4**2, 3, 1, 1),
|
||||
nn.PixelShuffle(4),
|
||||
)
|
||||
|
||||
def forward(self, value):
|
||||
value = self.to_feat(value)
|
||||
return self.to_img(self.feats(value) + value)
|
||||
|
||||
model = SAFMN()
|
||||
model.load_state_dict(
|
||||
_load_weights(torch, checkpoint_path, "params_ema"),
|
||||
strict=True,
|
||||
)
|
||||
sample = torch.rand(1, 3, 256, 256)
|
||||
_export(torch, model, sample, destination)
|
||||
_validate_export(onnx, onnxruntime, torch, model, sample, destination)
|
||||
|
||||
|
||||
def export_model(exporter, checkpoint_path, destination):
|
||||
exporters = {
|
||||
"realesrgan_wdn_x4": export_realesrgan_wdn_x4,
|
||||
"safmn_real_x4": export_safmn_real_x4,
|
||||
}
|
||||
try:
|
||||
export = exporters[exporter]
|
||||
except KeyError as error:
|
||||
raise ValueError(f"Unknown model exporter: {exporter}") from error
|
||||
export(checkpoint_path, destination)
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import bpy
|
||||
|
||||
from anyimage.operators.common.mesh_result import (
|
||||
from anyimage.common.object import (
|
||||
DEPTH_MATERIAL_NODE_GROUP_NAME,
|
||||
IMAGE_MATERIAL_NODE_GROUP_NAME,
|
||||
)
|
||||
|
||||
@@ -10,19 +10,20 @@ sys.path.insert(
|
||||
str(next((PROJECT_ROOT / "src" / "anyimage" / "wheels").glob("blendjob-*.whl"))),
|
||||
)
|
||||
|
||||
from anyimage.operators.common.mesh_result import (
|
||||
from anyimage.common.object import (
|
||||
DEPTH_MATERIAL_NODE_GROUP_NAME,
|
||||
IMAGE_MATERIAL_NODE_GROUP_NAME,
|
||||
load_asset_node_group,
|
||||
node_asset_path,
|
||||
)
|
||||
from anyimage.operators.convert_to_plane.nodes import DEPTH_PLANE_NODE_GROUP_NAME
|
||||
from anyimage.common.node import load_node_group, node_asset_path
|
||||
from anyimage.operators.cutout_tool.shape import (
|
||||
CUTOUT_NODE_GROUP_NAME,
|
||||
CUTOUT_NODE_GROUP_NAMES,
|
||||
)
|
||||
|
||||
|
||||
DEPTH_PLANE_NODE_GROUP_NAME = "O Image Depth Plane"
|
||||
|
||||
|
||||
def input_names(group):
|
||||
return [
|
||||
item.name
|
||||
@@ -33,12 +34,12 @@ def input_names(group):
|
||||
|
||||
def main():
|
||||
bpy.ops.wm.read_factory_settings(use_empty=True)
|
||||
mesh_plane = load_asset_node_group("O Mesh Plane")
|
||||
depth_plane = load_asset_node_group(DEPTH_PLANE_NODE_GROUP_NAME)
|
||||
image_layer = load_asset_node_group(IMAGE_MATERIAL_NODE_GROUP_NAME)
|
||||
depth_layer = load_asset_node_group(DEPTH_MATERIAL_NODE_GROUP_NAME)
|
||||
shadeless = load_asset_node_group("O Shadeless")
|
||||
cutout = load_asset_node_group(CUTOUT_NODE_GROUP_NAME)
|
||||
mesh_plane = load_node_group("O Mesh Plane")
|
||||
depth_plane = load_node_group(DEPTH_PLANE_NODE_GROUP_NAME)
|
||||
image_layer = load_node_group(IMAGE_MATERIAL_NODE_GROUP_NAME)
|
||||
depth_layer = load_node_group(DEPTH_MATERIAL_NODE_GROUP_NAME)
|
||||
shadeless = load_node_group("O Shadeless")
|
||||
cutout = load_node_group(CUTOUT_NODE_GROUP_NAME)
|
||||
cutout_groups = {
|
||||
name: bpy.data.node_groups[name] for name in CUTOUT_NODE_GROUP_NAMES.values()
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ sys.path.insert(
|
||||
str(next((PROJECT_ROOT / "src" / "anyimage" / "wheels").glob("blendjob-*.whl"))),
|
||||
)
|
||||
|
||||
from anyimage.operators.common.mesh_result import (
|
||||
from anyimage.common.object import (
|
||||
modifier_input_identifier,
|
||||
set_modifier_input,
|
||||
)
|
||||
|
||||
@@ -11,7 +11,7 @@ sys.path.insert(
|
||||
str(next((PROJECT_ROOT / "src" / "anyimage" / "wheels").glob("blendjob-*.whl"))),
|
||||
)
|
||||
|
||||
from anyimage.operators.common.mesh_result import (
|
||||
from anyimage.common.object import (
|
||||
modifier_input_identifier,
|
||||
set_modifier_input,
|
||||
)
|
||||
|
||||
145
tools/pack.py
Normal file
145
tools/pack.py
Normal file
@@ -0,0 +1,145 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from zipfile import ZIP_DEFLATED, ZipFile
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCE_DIR = PROJECT_ROOT / "src" / "anyimage"
|
||||
DIST_DIR = PROJECT_ROOT / "dist"
|
||||
PYTHON_VERSION = "3.11"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Platform:
|
||||
pip_platform: str
|
||||
|
||||
|
||||
PLATFORM_CONFIGS = {
|
||||
"windows-x64": Platform(pip_platform="win_amd64"),
|
||||
"macos-arm64": Platform(pip_platform="macosx_12_0_arm64"),
|
||||
"linux-x64": Platform(pip_platform="manylinux2014_x86_64"),
|
||||
}
|
||||
PLATFORMS = tuple(PLATFORM_CONFIGS)
|
||||
|
||||
|
||||
def project_dependencies():
|
||||
"""Return Extension wheel requirements declared by the project."""
|
||||
project = tomllib.loads(
|
||||
(PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8")
|
||||
)["project"]
|
||||
return project.get("dependencies", [])
|
||||
|
||||
|
||||
def download_wheels(platform, destination):
|
||||
"""Download default dependencies as wheels for one target platform."""
|
||||
dependencies = project_dependencies()
|
||||
if not dependencies:
|
||||
return []
|
||||
config = PLATFORM_CONFIGS[platform]
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"download",
|
||||
*dependencies,
|
||||
"--dest",
|
||||
str(destination),
|
||||
"--find-links",
|
||||
str(SOURCE_DIR / "wheels"),
|
||||
"--only-binary=:all:",
|
||||
f"--python-version={PYTHON_VERSION}",
|
||||
f"--platform={config.pip_platform}",
|
||||
]
|
||||
subprocess.run(command, check=True)
|
||||
wheels = sorted(destination.glob("*.whl"))
|
||||
if not wheels:
|
||||
raise RuntimeError(f"No wheels downloaded for {platform}")
|
||||
return wheels
|
||||
|
||||
|
||||
def manifest_for_platform(platform, wheels):
|
||||
"""Return manifest text for one platform and its downloaded wheels."""
|
||||
text = (SOURCE_DIR / "blender_manifest.toml").read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
table_start = next(
|
||||
(i for i, line in enumerate(lines) if line.startswith("[")),
|
||||
len(lines),
|
||||
)
|
||||
head = lines[:table_start]
|
||||
tail = lines[table_start:]
|
||||
head = [
|
||||
line
|
||||
for line in head
|
||||
if not line.strip().startswith(("platforms =", "wheels ="))
|
||||
]
|
||||
insert_at = next(
|
||||
(
|
||||
i + 1
|
||||
for i, line in enumerate(head)
|
||||
if line.strip().startswith("blender_version_min")
|
||||
),
|
||||
len(head),
|
||||
)
|
||||
head.insert(insert_at, f'platforms = ["{platform}"]')
|
||||
wheel_names = ", ".join(
|
||||
f'"./wheels/{wheel.name}"' for wheel in wheels
|
||||
)
|
||||
head.insert(insert_at + 1, f"wheels = [{wheel_names}]")
|
||||
return "\n".join(head + tail) + "\n"
|
||||
|
||||
|
||||
def _build_extension(platform, wheels):
|
||||
manifest_text = manifest_for_platform(platform, wheels)
|
||||
manifest = tomllib.loads(manifest_text)
|
||||
output_path = DIST_DIR / (
|
||||
f"{manifest['name']}.v{manifest['version']}.{platform}.zip"
|
||||
)
|
||||
DIST_DIR.mkdir(exist_ok=True)
|
||||
with ZipFile(output_path, "w", ZIP_DEFLATED) as archive:
|
||||
for source_path in sorted(SOURCE_DIR.rglob("*")):
|
||||
if not source_path.is_file():
|
||||
continue
|
||||
if "__pycache__" in source_path.parts:
|
||||
continue
|
||||
if (
|
||||
source_path.suffix.startswith(".blend")
|
||||
and source_path.suffix != ".blend"
|
||||
):
|
||||
continue
|
||||
relative_path = source_path.relative_to(SOURCE_DIR)
|
||||
if relative_path.parts[0] == "wheels":
|
||||
continue
|
||||
if source_path.name == "blender_manifest.toml":
|
||||
continue
|
||||
archive.write(source_path, relative_path)
|
||||
archive.writestr("blender_manifest.toml", manifest_text)
|
||||
for wheel in wheels:
|
||||
archive.write(wheel, Path("wheels") / wheel.name)
|
||||
return output_path
|
||||
|
||||
|
||||
def build_extension(platform):
|
||||
if platform not in PLATFORMS:
|
||||
raise ValueError(f"Unsupported platform: {platform}")
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
wheels = download_wheels(platform, Path(directory))
|
||||
return _build_extension(platform, wheels)
|
||||
|
||||
|
||||
def build_all():
|
||||
return [build_extension(platform) for platform in PLATFORMS]
|
||||
|
||||
|
||||
def main():
|
||||
extension_paths = build_all()
|
||||
for extension_path in extension_paths:
|
||||
size_mib = extension_path.stat().st_size / (1024 * 1024)
|
||||
print(f"Built {extension_path} ({size_mib:.2f} MiB)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
244
tools/sync_models.py
Normal file
244
tools/sync_models.py
Normal file
@@ -0,0 +1,244 @@
|
||||
"""Prepare AnyImage ONNX artifacts and upload them to OmooLab R2."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
MODELS_DIR = PROJECT_ROOT / "models"
|
||||
MODEL_CACHE_DIR = PROJECT_ROOT / ".model-cache"
|
||||
SOURCE_ROOT = PROJECT_ROOT / "src" / "anyimage"
|
||||
sys.path.insert(0, str(SOURCE_ROOT))
|
||||
from server.model_catalog import DOWNLOADABLE_MODELS
|
||||
|
||||
R2_REMOTE = "omoolab-r2:ai-models"
|
||||
CHUNK_SIZE = 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuildSpec:
|
||||
source_url: str
|
||||
source_filename: str
|
||||
source_size: int
|
||||
source_sha256: str
|
||||
exporter: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelFile:
|
||||
key: str
|
||||
folder: str
|
||||
filename: str
|
||||
size: int
|
||||
sha256: str
|
||||
source_url: str = ""
|
||||
build: BuildSpec | None = None
|
||||
|
||||
@property
|
||||
def destination(self):
|
||||
return MODELS_DIR / self.folder / self.filename
|
||||
|
||||
@property
|
||||
def remote(self):
|
||||
return f"{R2_REMOTE}/{self.folder}/{self.filename}"
|
||||
|
||||
|
||||
MODEL_BUILDS = {
|
||||
"REALESRGAN_GENERAL_WDN_X4V3": BuildSpec(
|
||||
source_url=(
|
||||
"https://github.com/xinntao/Real-ESRGAN/releases/download/"
|
||||
"v0.2.5.0/realesr-general-wdn-x4v3.pth"
|
||||
),
|
||||
source_filename="realesr-general-wdn-x4v3.pth",
|
||||
source_size=4885111,
|
||||
source_sha256=(
|
||||
"1641f8c4464b9f097c9fdda5589273713f67cf59f3d909e0bd688f0cee269dca"
|
||||
),
|
||||
exporter="realesrgan_wdn_x4",
|
||||
),
|
||||
"SAFMN_REAL_X4": BuildSpec(
|
||||
source_url=(
|
||||
"https://github.com/sunny2109/SAFMN/releases/download/"
|
||||
"v0.1.0/SAFMN_L_Real_LSDIR_x4-v2.pth"
|
||||
),
|
||||
source_filename="SAFMN_L_Real_LSDIR_x4-v2.pth",
|
||||
source_size=45002097,
|
||||
source_sha256=(
|
||||
"f1ac0ee3ee143fbbc49aff6584cb7b48b71ba8d1961dfb8ad076b98ca4799ae2"
|
||||
),
|
||||
exporter="safmn_real_x4",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def hf_url(repository, filename):
|
||||
return f"https://huggingface.co/{repository}/resolve/main/{filename}"
|
||||
|
||||
|
||||
def catalog_model_files():
|
||||
files = []
|
||||
for key, model in DOWNLOADABLE_MODELS.items():
|
||||
for filename, size, sha256 in model.r2_files:
|
||||
source_url = ""
|
||||
if model.huggingface_repository:
|
||||
source_url = hf_url(model.huggingface_repository, filename)
|
||||
files.append(
|
||||
ModelFile(
|
||||
key=key,
|
||||
folder=model.r2_directory,
|
||||
filename=filename,
|
||||
size=size,
|
||||
sha256=sha256,
|
||||
source_url=source_url,
|
||||
build=MODEL_BUILDS.get(key),
|
||||
)
|
||||
)
|
||||
return tuple(files)
|
||||
|
||||
|
||||
MODEL_FILES = catalog_model_files()
|
||||
|
||||
|
||||
def file_sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open("rb") as stream:
|
||||
while chunk := stream.read(CHUNK_SIZE):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def verify_file(path, size, sha256):
|
||||
path = Path(path)
|
||||
if not path.is_file():
|
||||
return False
|
||||
if path.stat().st_size != size:
|
||||
raise RuntimeError(f"File has an unexpected size: {path}")
|
||||
if file_sha256(path) != sha256:
|
||||
raise RuntimeError(f"File has an unexpected SHA-256: {path}")
|
||||
return True
|
||||
|
||||
|
||||
def download_file(url, destination, size, sha256):
|
||||
destination = Path(destination)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = destination.with_name(f"{destination.name}.part")
|
||||
temporary.unlink(missing_ok=True)
|
||||
request = Request(url, headers={"User-Agent": "AnyImage model build"})
|
||||
print(f"Download: {url}")
|
||||
downloaded = 0
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
with urlopen(request, timeout=60) as response, temporary.open("wb") as output:
|
||||
while chunk := response.read(CHUNK_SIZE):
|
||||
output.write(chunk)
|
||||
digest.update(chunk)
|
||||
downloaded += len(chunk)
|
||||
print(
|
||||
f"\r {downloaded / 1024**2:.1f} / {size / 1024**2:.1f} MiB",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
print()
|
||||
if downloaded != size:
|
||||
raise RuntimeError(f"size mismatch: {downloaded} != {size}")
|
||||
if digest.hexdigest() != sha256:
|
||||
raise RuntimeError("SHA-256 mismatch")
|
||||
temporary.replace(destination)
|
||||
except Exception:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def prepare_source(build):
|
||||
source = MODEL_CACHE_DIR / build.source_filename
|
||||
if verify_file(source, build.source_size, build.source_sha256):
|
||||
return source
|
||||
download_file(
|
||||
build.source_url,
|
||||
source,
|
||||
build.source_size,
|
||||
build.source_sha256,
|
||||
)
|
||||
return source
|
||||
|
||||
|
||||
def build_model(model):
|
||||
from tools.model_exports import export_model
|
||||
|
||||
source = prepare_source(model.build)
|
||||
destination = model.destination
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = destination.with_name(f"{destination.name}.part")
|
||||
temporary.unlink(missing_ok=True)
|
||||
print(f"Build: {destination.relative_to(PROJECT_ROOT)}")
|
||||
try:
|
||||
export_model(model.build.exporter, source, temporary)
|
||||
verify_file(temporary, model.size, model.sha256)
|
||||
temporary.replace(destination)
|
||||
except Exception:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def prepare_model(model):
|
||||
if verify_file(model.destination, model.size, model.sha256):
|
||||
print(f"Skip existing: {model.destination.relative_to(PROJECT_ROOT)}")
|
||||
return
|
||||
if model.build:
|
||||
build_model(model)
|
||||
return
|
||||
if not model.source_url:
|
||||
raise RuntimeError(f"No upstream source or builder for {model.key}")
|
||||
download_file(
|
||||
model.source_url,
|
||||
model.destination,
|
||||
model.size,
|
||||
model.sha256,
|
||||
)
|
||||
|
||||
|
||||
def upload_models(models=MODEL_FILES):
|
||||
executable = shutil.which("rclone")
|
||||
if executable is None:
|
||||
raise RuntimeError("rclone is not available on PATH")
|
||||
for model in models:
|
||||
print(f"Upload: {model.remote}")
|
||||
subprocess.run(
|
||||
[
|
||||
executable,
|
||||
"copyto",
|
||||
str(model.destination),
|
||||
model.remote,
|
||||
"--progress",
|
||||
],
|
||||
cwd=PROJECT_ROOT,
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--prepare-only",
|
||||
action="store_true",
|
||||
help="Prepare and verify models without uploading them",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
for model in MODEL_FILES:
|
||||
prepare_model(model)
|
||||
if not args.prepare_only:
|
||||
upload_models()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user