feat: Bump version to 0.11.0 and update dependencies

- Updated ONNX Runtime to 1.24.4 and replaced OpenCV with PyAV 17.0.0 in the installation environment.
- Enhanced model downloading logic to prioritize Hugging Face and fallback to OmooLab R2, with improved error handling and status reporting.
- Introduced a new environment revision file for shared metadata.
- Refactored video frame extraction to utilize the PyAV library instead of OpenCV.
- Improved installation process to handle retries across specified PyPI indexes.
- Added tests for download progress reporting and model downloading behavior.
This commit is contained in:
2026-08-14 14:47:40 +08:00
parent 2ad30bf4a3
commit 5255eb4e96
12 changed files with 561 additions and 304 deletions

View File

@@ -60,6 +60,29 @@ class DownloadProgressTest(unittest.TestCase):
self.assertAlmostEqual(status["progress"], 0.275)
self.assertEqual(status["message"], "Downloading owner/model")
def test_reports_byte_progress_for_aggregated_bar_description(self):
# huggingface_hub 1.x 用 "Downloading (incomplete total...)" 作为字节进度条描述
with tempfile.TemporaryDirectory() as directory:
status_path = Path(directory) / "status.json"
progress_class = self.worker.create_download_progress_class(
status_path,
"owner/model",
tqdm_type=FakeTqdm,
clock=lambda: 1.0,
)
progress = progress_class(
desc="Downloading (incomplete total...)",
total=100,
initial=0,
unit="B",
)
progress.update(25)
status = json.loads(status_path.read_text(encoding="utf-8"))
self.assertAlmostEqual(status["progress"], 0.275)
self.assertEqual(status["message"], "Downloading owner/model")
def test_reports_download_speed(self):
class FakeTqdmWithRate(FakeTqdm):
@property

View File

@@ -521,7 +521,59 @@ class InstallEnvTest(unittest.TestCase):
self.assertEqual(first.kwargs["stages"], 8)
self.assertEqual(first.kwargs["stage_label"], "pytorch deps")
def test_run_install_retries_once_with_mirror(self):
def test_run_install_heartbeat_reports_package_count(self):
bootstrap = load_install_env()
class SlowLines:
def __iter__(self):
yield "Resolved 3 packages in 1.0s\n"
time.sleep(0.6)
class FakeProcess:
def __init__(self, lines):
self.stdout = lines
def wait(self):
return 0
clock_state = {"now": 0.0}
def fake_clock():
clock_state["now"] += 6.0
return clock_state["now"]
fake_process = FakeProcess(SlowLines())
with patch.object(
bootstrap.subprocess,
"Popen",
return_value=fake_process,
), patch.object(bootstrap, "write_status") as mocked_write:
bootstrap._run_install_once(
"status.json",
2,
2,
"ONNX Runtime",
["uv", "pip", "install", "onnxruntime"],
None,
0.10,
0.85,
fake_clock,
)
heartbeats = [
call
for call in mocked_write.call_args_list
if call.args[2].startswith("Downloading")
]
self.assertTrue(heartbeats)
self.assertTrue(
any("3 packages" in call.args[2] for call in heartbeats)
)
for call in heartbeats:
self.assertNotIn("elapsed", call.args[2])
self.assertNotIn("eta_seconds", call.kwargs)
def test_run_install_retries_across_indexes(self):
bootstrap = load_install_env()
previous = bootstrap._resolved_pypi_index
bootstrap._resolved_pypi_index = None
@@ -546,32 +598,44 @@ class InstallEnvTest(unittest.TestCase):
self.assertEqual(installed, ["numpy==1.26.4"])
self.assertEqual(mocked_install.call_count, 2)
# 未指定源时默认国内镜像优先,第一个镜像失败后切到第二个镜像。
self.assertEqual(
mocked_install.call_args_list[0].args[5],
{"UV_INDEX_URL": bootstrap.PYPI_MIRRORS[0]},
)
self.assertEqual(
mocked_install.call_args_list[1].args[5],
{"PIP_INDEX_URL": bootstrap.PYPI_MIRRORS[0]},
{"UV_INDEX_URL": bootstrap.PYPI_MIRRORS[1]},
)
def test_formats_elapsed_time(self):
def test_run_install_uses_only_configured_index(self):
bootstrap = load_install_env()
self.assertEqual(bootstrap.format_elapsed(12), "12s")
self.assertEqual(bootstrap.format_elapsed(83), "1m 23s")
def test_onnx_runtime_estimate_decreases_with_progress(self):
bootstrap = load_install_env()
first = bootstrap.estimated_remaining(
"ONNX Runtime",
bootstrap.AI_COMPONENTS_SLICE[0],
)
second = bootstrap.estimated_remaining(
"ONNX Runtime",
sum(bootstrap.AI_COMPONENTS_SLICE) / 2,
previous = bootstrap._resolved_pypi_index
bootstrap._resolved_pypi_index = (
"https://pypi.tuna.tsinghua.edu.cn/simple"
)
try:
with patch.object(
bootstrap,
"_run_install_once",
return_value=["numpy==1.26.4"],
) as mocked_install:
installed = bootstrap.run_install(
"status.json",
3,
8,
"ai deps",
["uv", "pip", "install", "x"],
)
finally:
bootstrap._resolved_pypi_index = previous
self.assertEqual(first, bootstrap.INSTALL_ESTIMATES["ONNX Runtime"])
self.assertGreater(second, 0)
self.assertLess(second, first)
self.assertEqual(installed, ["numpy==1.26.4"])
self.assertEqual(mocked_install.call_count, 1)
self.assertEqual(
mocked_install.call_args.args[5],
{"UV_INDEX_URL": "https://pypi.tuna.tsinghua.edu.cn/simple"},
)
def test_resolve_indexes_uses_configured_pypi_index(self):
bootstrap = load_install_env()
@@ -584,6 +648,18 @@ class InstallEnvTest(unittest.TestCase):
)
def test_resolve_indexes_uses_environment_override(self):
bootstrap = load_install_env()
with patch.dict(
os.environ,
{"UV_INDEX_URL": "https://pypi.example/simple"},
):
bootstrap.resolve_indexes(SimpleNamespace(pypi_index=None))
self.assertEqual(
bootstrap._resolved_pypi_index,
"https://pypi.example/simple",
)
def test_resolve_indexes_falls_back_to_pip_index_url(self):
bootstrap = load_install_env()
with patch.dict(
os.environ,
@@ -607,7 +683,7 @@ class InstallEnvTest(unittest.TestCase):
bootstrap._resolved_pypi_index = previous_pypi
self.assertEqual(
env["PIP_INDEX_URL"],
env["UV_INDEX_URL"],
"https://pypi.tuna.tsinghua.edu.cn/simple",
)
self.assertEqual(

View File

@@ -1,6 +1,7 @@
import importlib.util
import json
import os
import shutil
import sys
import tempfile
import unittest
@@ -25,6 +26,49 @@ def load_worker():
return module
def _make_fake_av(frame_count):
"""Build a fake av module yielding frame_count integer-valued frames."""
class FakeImage:
def __init__(self, value):
self.value = value
def save(self, path, format=None):
Path(path).write_text(str(self.value), encoding="utf-8")
class FakeFrame:
def __init__(self, value):
self.value = value
def to_image(self):
return FakeImage(self.value)
class FakeContainer:
def __init__(self):
self.position = 0
self.closed = False
self.streams = SimpleNamespace(video=[object()])
def decode(self, _stream):
while self.position < frame_count:
frame = FakeFrame(self.position)
self.position += 1
yield frame
def close(self):
self.closed = True
class FakeAv:
def __init__(self):
self.containers = []
def open(self, _path):
container = FakeContainer()
self.containers.append(container)
return container
return FakeAv()
class WorkerTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
@@ -177,92 +221,32 @@ class WorkerTest(unittest.TestCase):
self.assertIsNone(temporary_directory)
def test_extract_video_frames_reads_frames_sequentially(self):
class FakeCapture:
def __init__(self):
self.position = 0
def isOpened(self):
return True
def read(self):
if self.position >= 100:
return False, None
frame = self.position
self.position += 1
return True, frame
def release(self):
return None
class FakeCv2:
def __init__(self):
self.captures = []
def VideoCapture(self, _path):
capture = FakeCapture()
self.captures.append(capture)
return capture
def imwrite(self, path, frame):
Path(path).write_text(str(frame), encoding="utf-8")
return True
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
fake_cv2 = FakeCv2()
fake_av = _make_fake_av(100)
image_paths = self.worker.extract_video_frames(
root / "clip.mkv",
root / "out",
10,
fake_cv2,
fake_av,
)
self.assertEqual(
[path.name for path in image_paths],
[f"frame.{index:04d}.jpg" for index in range(1, 11)],
)
self.assertEqual(len(fake_cv2.captures), 1)
self.assertEqual(fake_cv2.captures[0].position, 10)
self.assertEqual(len(fake_av.containers), 1)
self.assertEqual(fake_av.containers[0].position, 10)
def test_extract_video_frames_starts_after_offset(self):
class FakeCapture:
def __init__(self):
self.position = 0
def isOpened(self):
return True
def read(self):
if self.position >= 100:
return False, None
frame = self.position
self.position += 1
return True, frame
def release(self):
return None
class FakeCv2:
def __init__(self):
self.captures = []
def VideoCapture(self, _path):
capture = FakeCapture()
self.captures.append(capture)
return capture
def imwrite(self, path, frame):
Path(path).write_text(str(frame), encoding="utf-8")
return True
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
fake_cv2 = FakeCv2()
fake_av = _make_fake_av(100)
image_paths = self.worker.extract_video_frames(
root / "clip.mkv",
root / "out",
10,
fake_cv2,
fake_av,
start_frame=71,
)
extracted = [
@@ -271,41 +255,16 @@ class WorkerTest(unittest.TestCase):
]
self.assertEqual(extracted, [str(frame) for frame in range(70, 80)])
self.assertEqual(fake_cv2.captures[0].position, 80)
self.assertEqual(fake_av.containers[0].position, 80)
def test_extract_video_frames_tolerates_short_video_tail(self):
class ShortCapture:
def __init__(self):
self.position = 0
def isOpened(self):
return True
def read(self):
if self.position >= 99:
return False, None
frame = self.position
self.position += 1
return True, frame
def release(self):
return None
class FakeCv2:
def VideoCapture(self, _path):
return ShortCapture()
def imwrite(self, path, frame):
Path(path).write_text(str(frame), encoding="utf-8")
return True
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
image_paths = self.worker.extract_video_frames(
root / "clip.mkv",
root / "out",
10,
FakeCv2(),
_make_fake_av(99),
)
self.assertEqual(
@@ -443,38 +402,120 @@ class WorkerTest(unittest.TestCase):
self.worker.download_model(args)
def test_download_model_reports_all_failed_sources(self):
fake_hugging_face = ModuleType("huggingface_hub")
def fake_snapshot_download(**options):
raise OSError("direct down")
fake_hugging_face.snapshot_download = fake_snapshot_download
args = SimpleNamespace(
repository="depth-anything/DA3-SMALL",
model_dir="model",
status="status.json",
allow_pattern=[],
r2_directory="DA3Small",
r2_file=["config.json|100|abc"],
)
with tempfile.TemporaryDirectory() as directory, patch.dict(
sys.modules,
{"huggingface_hub": fake_hugging_face},
), patch.object(
with tempfile.TemporaryDirectory() as directory, patch.object(
self.worker,
"create_download_progress_class",
return_value=object,
), patch.dict(os.environ, {"HF_ENDPOINT": ""}), patch.object(
"download_from_hf",
side_effect=OSError("direct down"),
), patch.object(
self.worker,
"download_from_r2",
side_effect=OSError("mirror down"),
):
args.r2_directory = "DA3Small"
args.r2_file = ["config.json|100|abc"]
args.model_dir = str(Path(directory) / "model")
args.status = str(Path(directory) / "status.json")
with self.assertRaisesRegex(RuntimeError, "models.omoolab.xyz"):
self.worker.download_model(args)
def test_download_model_prefers_direct_hf_then_r2(self):
args = SimpleNamespace(
repository="depth-anything/DA3-SMALL",
model_dir="model",
status="status.json",
allow_pattern=["config.json"],
r2_directory="DA3Small",
r2_file=["config.json|100|abc"],
)
order = []
def fake_download_from_hf(*_args, **_kwargs):
order.append("hf")
raise OSError("hf down")
def fake_download_from_r2(*_args, **_kwargs):
order.append("r2")
with tempfile.TemporaryDirectory() as directory, patch.object(
self.worker,
"download_from_hf",
side_effect=fake_download_from_hf,
), patch.object(
self.worker,
"download_from_r2",
side_effect=fake_download_from_r2,
), patch.object(
self.worker,
"_verify_model_complete",
return_value=None,
):
args.model_dir = str(Path(directory) / "model")
args.status = str(Path(directory) / "status.json")
self.worker.download_model(args)
self.assertEqual(order, ["hf", "r2"])
def test_download_declared_skips_hf_when_r2_preferred(self):
root = PROJECT_ROOT / ".tmp-models-from-r2"
model_dir = root / "models" / "DA3"
args = SimpleNamespace(
repository="depth-anything/DA3-SMALL",
model_dir=str(model_dir),
status="status.json",
allow_pattern=[],
r2_directory="DA3Small",
r2_file=["config.json|100|abc"],
)
order = []
try:
self.worker._mark_models_from_r2(model_dir)
with patch.object(
self.worker,
"download_from_hf",
side_effect=lambda *a, **k: order.append("hf"),
), patch.object(
self.worker,
"download_from_r2",
side_effect=lambda *a, **k: order.append("r2"),
), patch.object(
self.worker,
"_verify_model_complete",
return_value=None,
), patch.object(self.worker, "write_status"):
self.worker._download_declared_or_fail(
args, model_dir, "DA3 Small", 0.0, 1.0, {}
)
finally:
shutil.rmtree(root, ignore_errors=True)
self.assertEqual(order, ["r2"])
def test_models_from_r2_roundtrip_preserves_fields(self):
root = PROJECT_ROOT / ".tmp-models-from-r2"
model_dir = root / "models" / "DA3"
discovery = root / "service.json"
try:
discovery.parent.mkdir(parents=True, exist_ok=True)
discovery.write_text(
json.dumps({"port": 1, "token": "t"}),
encoding="utf-8",
)
self.assertFalse(self.worker._models_from_r2(model_dir))
self.worker._mark_models_from_r2(model_dir)
self.assertTrue(self.worker._models_from_r2(model_dir))
payload = json.loads(discovery.read_text(encoding="utf-8"))
self.assertEqual(payload["port"], 1)
self.assertTrue(payload["models_from_r2"])
finally:
shutil.rmtree(root, ignore_errors=True)
def test_download_from_r2_downloads_declared_files(self):
args = SimpleNamespace(
repository="depth-anything/DA3-SMALL",
@@ -488,8 +529,8 @@ class WorkerTest(unittest.TestCase):
with tempfile.TemporaryDirectory() as directory, patch.object(
self.worker,
"_download_r2_file",
side_effect=lambda *args, **kwargs: downloaded.append(args[1]),
"_download_file_from_url",
side_effect=lambda *args, **kwargs: downloaded.append(args[1].name),
), patch.object(self.worker, "write_status"):
args.model_dir = str(Path(directory) / "model")
args.status = str(Path(directory) / "status.json")