Add reusable local HTTP job service for Blender add-ons
- Implement JobClient for handling HTTP requests to the JobServer. - Create ModalJobOperatorBase for managing Blender job lifecycle. - Introduce JobContext and JobServer for job management and state handling. - Add ServiceConnection and ServiceController for managing service processes. - Implement job submission, cancellation, and resource management endpoints in the server. - Ensure proper error handling and job state updates throughout the service.
This commit is contained in:
37
src/snapform/blendjob/__init__.py
Normal file
37
src/snapform/blendjob/__init__.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""Reusable local HTTP job service for Blender add-ons."""
|
||||
|
||||
from .client import JobClient, JobRequest
|
||||
from .server import JobCancelled, JobContext, JobServer
|
||||
from .service import ServiceConnection, ServiceController
|
||||
|
||||
try:
|
||||
from .operator import (
|
||||
ModalJobOperatorBase,
|
||||
ModalJobState,
|
||||
active_job,
|
||||
cancel_active_job,
|
||||
close_active_job,
|
||||
)
|
||||
except ModuleNotFoundError as error:
|
||||
if error.name != "bpy":
|
||||
raise
|
||||
|
||||
|
||||
__all__ = (
|
||||
"JobCancelled",
|
||||
"JobClient",
|
||||
"JobContext",
|
||||
"JobRequest",
|
||||
"JobServer",
|
||||
"ServiceConnection",
|
||||
"ServiceController",
|
||||
)
|
||||
|
||||
if "ModalJobOperatorBase" in globals():
|
||||
__all__ += (
|
||||
"ModalJobOperatorBase",
|
||||
"ModalJobState",
|
||||
"active_job",
|
||||
"cancel_active_job",
|
||||
"close_active_job",
|
||||
)
|
||||
99
src/snapform/blendjob/client.py
Normal file
99
src/snapform/blendjob/client.py
Normal file
@@ -0,0 +1,99 @@
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import ProxyHandler, Request, build_opener
|
||||
|
||||
|
||||
class JobClient:
|
||||
"""Dependency-free HTTP client for a local :class:`JobServer`."""
|
||||
|
||||
def __init__(self, host, port, *, timeout=2.0):
|
||||
self.base_url = f"http://{host}:{int(port)}"
|
||||
self.timeout = timeout
|
||||
self.opener = build_opener(ProxyHandler({}))
|
||||
|
||||
def request(self, method, path, payload=None, timeout=None):
|
||||
data = None
|
||||
headers = {}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = Request(
|
||||
f"{self.base_url}{path}",
|
||||
data=data,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with self.opener.open(
|
||||
request,
|
||||
timeout=self.timeout if timeout is None else timeout,
|
||||
) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except HTTPError as error:
|
||||
try:
|
||||
detail = json.loads(error.read().decode("utf-8")).get("detail")
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
detail = None
|
||||
raise RuntimeError(detail or f"Job Server returned HTTP {error.code}")
|
||||
except (OSError, URLError) as error:
|
||||
raise RuntimeError(f"Unable to reach Job Server: {error}")
|
||||
|
||||
def health(self):
|
||||
return self.request("GET", "/health")
|
||||
|
||||
def submit(self, job_type, **parameters):
|
||||
return self.request(
|
||||
"POST",
|
||||
"/jobs",
|
||||
{
|
||||
"job_type": job_type,
|
||||
"parameters": parameters,
|
||||
},
|
||||
)
|
||||
|
||||
def submit_request(self, request):
|
||||
return self.request("POST", "/jobs", request.payload)
|
||||
|
||||
def status(self, job_id):
|
||||
return self.request("GET", f"/jobs/{job_id}")
|
||||
|
||||
def cancel(self, job_id):
|
||||
return self.request("DELETE", f"/jobs/{job_id}")
|
||||
|
||||
def resources(self):
|
||||
return self.request("GET", "/resources")
|
||||
|
||||
def resource(self, name):
|
||||
return self.request("GET", f"/resources/{name}")
|
||||
|
||||
def clear_resource(self, name):
|
||||
return self.request("POST", f"/resources/{name}/clear")
|
||||
|
||||
def shutdown(self):
|
||||
return self.request("POST", "/shutdown")
|
||||
@dataclass(frozen=True, init=False)
|
||||
class JobRequest:
|
||||
"""Serializable request created before a Job is submitted."""
|
||||
|
||||
job_type: str
|
||||
parameters: dict
|
||||
|
||||
def __init__(self, job_type, parameters=None):
|
||||
if isinstance(job_type, dict):
|
||||
payload = job_type
|
||||
job_type = payload.get("job_type", "")
|
||||
parameters = payload.get("parameters", {})
|
||||
object.__setattr__(self, "job_type", str(job_type))
|
||||
object.__setattr__(self, "parameters", dict(parameters or {}))
|
||||
|
||||
@classmethod
|
||||
def create(cls, job_type, **parameters):
|
||||
return cls(job_type, parameters)
|
||||
|
||||
@property
|
||||
def payload(self):
|
||||
return {
|
||||
"job_type": self.job_type,
|
||||
"parameters": self.parameters,
|
||||
}
|
||||
267
src/snapform/blendjob/operator.py
Normal file
267
src/snapform/blendjob/operator.py
Normal file
@@ -0,0 +1,267 @@
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import bpy
|
||||
|
||||
from .client import JobRequest
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModalJobState:
|
||||
"""Runtime state shared by all Blender modal Job operators."""
|
||||
|
||||
controller: object
|
||||
data: object = None
|
||||
cleanup: object = None
|
||||
job_id: str = ""
|
||||
directory: Path | None = None
|
||||
started: bool = False
|
||||
cancelled: bool = False
|
||||
start_error: str = ""
|
||||
start_thread: threading.Thread | None = None
|
||||
progress_stage: int | None = None
|
||||
progress: float = 0.0
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name == "result_directory":
|
||||
return self.directory
|
||||
if self.data is None:
|
||||
raise AttributeError(name)
|
||||
return getattr(self.data, name)
|
||||
|
||||
|
||||
_active_job = None
|
||||
|
||||
|
||||
def active_job():
|
||||
return _active_job
|
||||
|
||||
|
||||
def cancel_active_job():
|
||||
job = active_job()
|
||||
if job is None:
|
||||
return
|
||||
job.cancelled = True
|
||||
if not job.started or not job.job_id:
|
||||
return
|
||||
try:
|
||||
job.controller.cancel(job.job_id)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
|
||||
def close_active_job():
|
||||
global _active_job
|
||||
|
||||
if _active_job is not None:
|
||||
if _active_job.job_id:
|
||||
_active_job.controller.mark_job_complete(_active_job.job_id)
|
||||
if _active_job.cleanup is not None:
|
||||
_active_job.cleanup(_active_job.data)
|
||||
_active_job.cleanup = None
|
||||
_active_job = None
|
||||
|
||||
|
||||
class ModalJobOperatorBase(bpy.types.Operator):
|
||||
"""Ready-to-use Blender lifecycle for one HTTP Job request."""
|
||||
|
||||
bl_options = {"INTERNAL"}
|
||||
poll_interval = 0.25
|
||||
starting_message = "Starting task"
|
||||
controller_factory = None
|
||||
job_type = ""
|
||||
|
||||
def get_service_controller(self):
|
||||
"""Return the ServiceController used by this add-on."""
|
||||
if self.controller_factory is None:
|
||||
raise RuntimeError("Set controller_factory on the Job operator")
|
||||
return self.controller_factory()
|
||||
|
||||
def create_job_request(self, context):
|
||||
"""Create the JobRequest for this operator invocation."""
|
||||
if not self.job_type:
|
||||
raise RuntimeError("Set job_type on the Job operator")
|
||||
return JobRequest(self.job_type, self.job_parameters(context))
|
||||
|
||||
def job_parameters(self, _context):
|
||||
"""Return JSON-compatible business parameters for this invocation."""
|
||||
return {}
|
||||
|
||||
def create_job_data(self, _context):
|
||||
"""Return optional plug-in data retained until completion."""
|
||||
return None
|
||||
|
||||
def execute(self, context):
|
||||
global _active_job
|
||||
|
||||
if active_job() is not None:
|
||||
self.report({"WARNING"}, "Another task is running")
|
||||
return {"CANCELLED"}
|
||||
try:
|
||||
request = self.create_job_request(context)
|
||||
if not isinstance(request, JobRequest):
|
||||
raise TypeError("create_job_request() must return JobRequest")
|
||||
controller = self.get_service_controller()
|
||||
data = self.create_job_data(context)
|
||||
except (OSError, RuntimeError, TypeError, ValueError) as error:
|
||||
self.report({"ERROR"}, str(error))
|
||||
return {"CANCELLED"}
|
||||
|
||||
job = ModalJobState(
|
||||
controller=controller,
|
||||
data=data,
|
||||
cleanup=self.cleanup_job_data,
|
||||
)
|
||||
_active_job = job
|
||||
if hasattr(context.window_manager, "progress_begin"):
|
||||
context.window_manager.progress_begin(0, 100)
|
||||
self.update_job_ui(context, 0.0, self.get_starting_message(context))
|
||||
self._timer = context.window_manager.event_timer_add(
|
||||
self.poll_interval,
|
||||
window=context.window,
|
||||
)
|
||||
context.window_manager.modal_handler_add(self)
|
||||
self.redraw_job_ui(context, force=True)
|
||||
job.start_thread = threading.Thread(
|
||||
target=self._submit_job,
|
||||
args=(job, request),
|
||||
name="BlendJobSubmit",
|
||||
daemon=True,
|
||||
)
|
||||
job.start_thread.start()
|
||||
return {"RUNNING_MODAL"}
|
||||
|
||||
def _submit_job(self, job, request):
|
||||
if job.cancelled:
|
||||
job.started = True
|
||||
return
|
||||
try:
|
||||
submitted = job.controller.submit(request)
|
||||
job.job_id = submitted["job_id"]
|
||||
job.directory = Path(submitted["directory"])
|
||||
job.started = True
|
||||
if job.cancelled:
|
||||
job.controller.cancel(job.job_id)
|
||||
except (KeyError, OSError, RuntimeError, TypeError) as error:
|
||||
job.start_error = str(error)
|
||||
|
||||
def modal(self, context, event):
|
||||
if event.type == "ESC":
|
||||
cancel_active_job()
|
||||
if event.type != "TIMER":
|
||||
return {"PASS_THROUGH"}
|
||||
|
||||
job = active_job()
|
||||
if job is None:
|
||||
self.remove_job_timer(context)
|
||||
return {"CANCELLED"}
|
||||
if job.start_error:
|
||||
return self._finish_start_error(context, job)
|
||||
if not job.started:
|
||||
return {"RUNNING_MODAL"}
|
||||
|
||||
try:
|
||||
status = job.controller.status(job.job_id)
|
||||
except RuntimeError as error:
|
||||
return self._finish_failure(context, job, str(error))
|
||||
self.update_from_job_status(context, job, status)
|
||||
state = status.get("state")
|
||||
if state == "succeeded":
|
||||
return self._finish_success(context, job, status)
|
||||
if state in {"failed", "cancelled"}:
|
||||
return self._finish_terminal(context, job, status)
|
||||
return {"RUNNING_MODAL"}
|
||||
|
||||
def _finish_start_error(self, context, job):
|
||||
self.report({"ERROR"}, f"Unable to start task: {job.start_error}")
|
||||
return self._close(context, job, "Task failed to start", cancelled=True)
|
||||
|
||||
def _finish_terminal(self, context, job, status):
|
||||
if job.cancelled or status.get("state") == "cancelled":
|
||||
return self._close(context, job, "Task cancelled", cancelled=True)
|
||||
return self._finish_failure(
|
||||
context,
|
||||
job,
|
||||
status.get("error") or "Service task failed",
|
||||
)
|
||||
|
||||
def _finish_failure(self, context, job, error):
|
||||
self.report({"ERROR"}, str(error))
|
||||
return self._close(context, job, "Task failed", cancelled=True)
|
||||
|
||||
def _finish_success(self, context, job, status):
|
||||
try:
|
||||
message = self.on_job_succeeded(context, job, status)
|
||||
except (OSError, RuntimeError, ValueError) as error:
|
||||
return self._finish_failure(context, job, error)
|
||||
return self._close(
|
||||
context,
|
||||
job,
|
||||
message or status.get("message") or "Task complete",
|
||||
cancelled=False,
|
||||
)
|
||||
|
||||
def _close(self, context, job, message, *, cancelled):
|
||||
self.remove_job_timer(context)
|
||||
if hasattr(context.window_manager, "progress_end"):
|
||||
context.window_manager.progress_end()
|
||||
close_active_job()
|
||||
self.update_job_ui(context, 0.0 if cancelled else 1.0, message)
|
||||
self.redraw_job_ui(context, force=True)
|
||||
return {"CANCELLED"} if cancelled else {"FINISHED"}
|
||||
|
||||
def get_starting_message(self, _context):
|
||||
return self.starting_message
|
||||
|
||||
def update_from_job_status(self, context, job, status):
|
||||
reported = min(max(float(status.get("progress", job.progress)), 0.0), 1.0)
|
||||
stage = status.get("stage")
|
||||
if stage is not None and stage != job.progress_stage:
|
||||
job.progress_stage = stage
|
||||
job.progress = reported
|
||||
else:
|
||||
job.progress = max(job.progress, reported)
|
||||
message = self.format_job_status(status, self.starting_message)
|
||||
self.update_job_ui(context, job.progress, message)
|
||||
self.redraw_job_ui(context, force=True)
|
||||
|
||||
def update_job_ui(self, context, progress, _message):
|
||||
"""Update Blender's standard progress indicator by default."""
|
||||
if hasattr(context.window_manager, "progress_update"):
|
||||
context.window_manager.progress_update(int(progress * 100))
|
||||
|
||||
def on_job_succeeded(self, _context, _job, status):
|
||||
"""Import results if needed and optionally return a final message."""
|
||||
return status.get("message")
|
||||
|
||||
def cleanup_job_data(self, _data):
|
||||
pass
|
||||
|
||||
def redraw_job_ui(self, _context, force=False):
|
||||
del force
|
||||
|
||||
def remove_job_timer(self, context):
|
||||
timer = getattr(self, "_timer", None)
|
||||
if timer is None:
|
||||
return
|
||||
context.window_manager.event_timer_remove(timer)
|
||||
self._timer = None
|
||||
|
||||
def format_job_status(self, status, fallback):
|
||||
message = status.get("message")
|
||||
stage = status.get("stage")
|
||||
stages = status.get("stages")
|
||||
stage_label = status.get("stage_label")
|
||||
if stage and stages:
|
||||
prefix = f"{stage}/{stages}"
|
||||
normalized = str(message or "").strip()
|
||||
if normalized and normalized.lower() != str(stage_label or "").lower():
|
||||
return f"{prefix} {normalized}"
|
||||
return f"{prefix} {stage_label}" if stage_label else prefix
|
||||
return str(message or stage_label or fallback).strip()
|
||||
|
||||
def cancel(self, context):
|
||||
cancel_active_job()
|
||||
self.remove_job_timer(context)
|
||||
self.redraw_job_ui(context, force=True)
|
||||
350
src/snapform/blendjob/server.py
Normal file
350
src/snapform/blendjob/server.py
Normal file
@@ -0,0 +1,350 @@
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
TERMINAL_STATES = frozenset({"succeeded", "failed", "cancelled"})
|
||||
|
||||
|
||||
class JobCancelled(RuntimeError):
|
||||
"""Raised by a handler when its job has been cancelled."""
|
||||
|
||||
|
||||
class JobContext:
|
||||
"""Mutable state and cancellation API passed to a registered job handler."""
|
||||
|
||||
def __init__(self, job_id, job_type, parameters, resources, directory):
|
||||
self.job_id = job_id
|
||||
self.job_type = job_type
|
||||
self.parameters = parameters
|
||||
self.directory = Path(directory)
|
||||
self._resources = resources
|
||||
self._lock = threading.Lock()
|
||||
self._cancel_requested = False
|
||||
self._status = {
|
||||
"job_id": job_id,
|
||||
"job_type": job_type,
|
||||
"state": "queued",
|
||||
"progress": 0.0,
|
||||
"message": "Task queued",
|
||||
"directory": str(self.directory),
|
||||
}
|
||||
|
||||
def update(self, progress, message, error=None, state=None, **details):
|
||||
"""Publish a progress snapshot returned by ``GET /jobs/{job_id}``."""
|
||||
if state is None:
|
||||
if error:
|
||||
state = "failed"
|
||||
elif progress >= 1.0:
|
||||
state = "succeeded"
|
||||
else:
|
||||
state = "running"
|
||||
snapshot = {
|
||||
"job_id": self.job_id,
|
||||
"job_type": self.job_type,
|
||||
"state": state,
|
||||
"progress": min(max(float(progress), 0.0), 1.0),
|
||||
"message": str(message),
|
||||
"directory": str(self.directory),
|
||||
**{key: value for key, value in details.items() if value is not None},
|
||||
}
|
||||
if error:
|
||||
snapshot["error"] = str(error)
|
||||
with self._lock:
|
||||
self._status = snapshot
|
||||
|
||||
def snapshot(self):
|
||||
with self._lock:
|
||||
return dict(self._status)
|
||||
|
||||
def request_cancel(self):
|
||||
with self._lock:
|
||||
if self._status["state"] in TERMINAL_STATES:
|
||||
return False
|
||||
self._cancel_requested = True
|
||||
self._status = {
|
||||
**self._status,
|
||||
"state": "cancelling",
|
||||
"message": "Cancelling task",
|
||||
}
|
||||
return True
|
||||
|
||||
def is_cancelled(self):
|
||||
with self._lock:
|
||||
return self._cancel_requested
|
||||
|
||||
def check_cancelled(self):
|
||||
if self.is_cancelled():
|
||||
raise JobCancelled("Task cancelled")
|
||||
|
||||
def resource(self, name):
|
||||
"""Return a shared resource registered on the owning JobServer."""
|
||||
try:
|
||||
return self._resources[name]
|
||||
except KeyError:
|
||||
raise KeyError(f"Unknown service resource: {name}") from None
|
||||
|
||||
|
||||
class JobServer:
|
||||
"""Small FastAPI wrapper with decorator-based job registration."""
|
||||
|
||||
def __init__(self, name, *, jobs_directory=None, max_job_history=100):
|
||||
self.name = name
|
||||
self.max_job_history = max(int(max_job_history), 0)
|
||||
self.handlers = {}
|
||||
self.resources = {}
|
||||
self.jobs = {}
|
||||
self.active_job_id = None
|
||||
self.lock = threading.Lock()
|
||||
self.shutdown_event = threading.Event()
|
||||
self.started_at = time.time()
|
||||
self.jobs_directory = Path(jobs_directory or Path.cwd() / "jobs")
|
||||
|
||||
def add_resource(self, name, resource):
|
||||
"""Register shared state owned for the full service lifetime."""
|
||||
if not name or "/" in name:
|
||||
raise ValueError(f"Invalid resource name: {name}")
|
||||
with self.lock:
|
||||
if name in self.resources:
|
||||
raise ValueError(f"Resource is already registered: {name}")
|
||||
self.resources[name] = resource
|
||||
return resource
|
||||
|
||||
def resource_snapshot(self, name):
|
||||
try:
|
||||
resource = self.resources[name]
|
||||
except KeyError:
|
||||
raise KeyError(name) from None
|
||||
snapshot = getattr(resource, "snapshot", None)
|
||||
return snapshot() if snapshot is not None else {}
|
||||
|
||||
def resource_snapshots(self):
|
||||
return {
|
||||
name: self.resource_snapshot(name)
|
||||
for name in tuple(self.resources)
|
||||
}
|
||||
|
||||
def clear_resource(self, name):
|
||||
"""Clear an idle service resource without racing a new job submission."""
|
||||
with self.lock:
|
||||
if self.active_job_id is not None:
|
||||
return False
|
||||
try:
|
||||
resource = self.resources[name]
|
||||
except KeyError:
|
||||
raise KeyError(name) from None
|
||||
clear = getattr(resource, "clear", None)
|
||||
if clear is None:
|
||||
raise TypeError(f"Resource cannot be cleared: {name}")
|
||||
clear()
|
||||
return True
|
||||
|
||||
def close(self):
|
||||
for resource in reversed(tuple(self.resources.values())):
|
||||
close = getattr(resource, "close", None)
|
||||
if close is not None:
|
||||
close()
|
||||
|
||||
def job(self, job_type):
|
||||
"""Register ``handler(context, parameters)`` for a public job type."""
|
||||
def register(handler):
|
||||
if job_type in self.handlers:
|
||||
raise ValueError(f"Job type is already registered: {job_type}")
|
||||
self.handlers[job_type] = handler
|
||||
return handler
|
||||
|
||||
return register
|
||||
|
||||
def snapshot(self, instance_id):
|
||||
with self.lock:
|
||||
active = self.jobs.get(self.active_job_id)
|
||||
result = {
|
||||
"ready": True,
|
||||
"service": self.name,
|
||||
"instance_id": instance_id,
|
||||
"busy": active is not None,
|
||||
"active_job": active.snapshot() if active else None,
|
||||
"started_at": self.started_at,
|
||||
"resources": self.resource_snapshots(),
|
||||
}
|
||||
return result
|
||||
|
||||
def submit(self, job_type, parameters, *, job_id=None):
|
||||
job_id = job_id or uuid.uuid4().hex
|
||||
if job_type not in self.handlers:
|
||||
raise KeyError(job_type)
|
||||
if not job_id or Path(job_id).name != job_id or job_id in {".", ".."}:
|
||||
raise ValueError(f"Invalid job id: {job_id}")
|
||||
with self.lock:
|
||||
if self.active_job_id is not None:
|
||||
return None
|
||||
if job_id in self.jobs:
|
||||
raise ValueError(f"Job id already exists: {job_id}")
|
||||
self.jobs_directory.mkdir(parents=True, exist_ok=True)
|
||||
directory = self.jobs_directory / job_id
|
||||
try:
|
||||
directory.mkdir()
|
||||
except FileExistsError:
|
||||
raise ValueError(f"Job directory already exists: {job_id}") from None
|
||||
context = JobContext(
|
||||
job_id,
|
||||
job_type,
|
||||
parameters,
|
||||
self.resources,
|
||||
directory,
|
||||
)
|
||||
self.jobs[job_id] = context
|
||||
self.active_job_id = job_id
|
||||
self._prune_jobs()
|
||||
return context
|
||||
|
||||
def _prune_jobs(self):
|
||||
completed = [
|
||||
identifier
|
||||
for identifier, context in self.jobs.items()
|
||||
if identifier != self.active_job_id
|
||||
and context.snapshot()["state"] in TERMINAL_STATES
|
||||
]
|
||||
excess = len(completed) - self.max_job_history
|
||||
for identifier in completed[:max(excess, 0)]:
|
||||
self.jobs.pop(identifier, None)
|
||||
|
||||
def run(self, context):
|
||||
context.update(0.0, "Service accepted the task", state="running")
|
||||
try:
|
||||
context.check_cancelled()
|
||||
self.handlers[context.job_type](context, context.parameters)
|
||||
context.check_cancelled()
|
||||
if context.snapshot()["state"] not in TERMINAL_STATES:
|
||||
context.update(1.0, "Task complete", state="succeeded")
|
||||
except JobCancelled:
|
||||
context.update(1.0, "Task cancelled", state="cancelled")
|
||||
except Exception as error:
|
||||
traceback.print_exc()
|
||||
context.update(1.0, "Task failed", error=error, state="failed")
|
||||
finally:
|
||||
with self.lock:
|
||||
if self.active_job_id == context.job_id:
|
||||
self.active_job_id = None
|
||||
|
||||
def create_app(self, instance_id):
|
||||
from fastapi import BackgroundTasks, FastAPI, HTTPException
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app):
|
||||
yield
|
||||
self.close()
|
||||
|
||||
app = FastAPI(
|
||||
title=self.name,
|
||||
docs_url=None,
|
||||
redoc_url=None,
|
||||
openapi_url=None,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
app.state.job_server = self
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return self.snapshot(instance_id)
|
||||
|
||||
@app.get("/resources")
|
||||
def resources():
|
||||
return self.resource_snapshots()
|
||||
|
||||
@app.get("/resources/{resource_name}")
|
||||
def resource_status(
|
||||
resource_name: str,
|
||||
):
|
||||
try:
|
||||
return self.resource_snapshot(resource_name)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail="Resource was not found")
|
||||
|
||||
@app.post("/resources/{resource_name}/clear")
|
||||
def clear_resource(
|
||||
resource_name: str,
|
||||
):
|
||||
try:
|
||||
cleared = self.clear_resource(resource_name)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail="Resource was not found")
|
||||
except TypeError as error:
|
||||
raise HTTPException(status_code=405, detail=str(error))
|
||||
if not cleared:
|
||||
raise HTTPException(status_code=409, detail="Service is busy")
|
||||
return {"resource": resource_name, "state": "cleared"}
|
||||
|
||||
@app.post("/jobs", status_code=202)
|
||||
def submit_job(
|
||||
payload: dict,
|
||||
background_tasks: BackgroundTasks,
|
||||
):
|
||||
job_type = str(payload.get("job_type", payload.get("command", "")))
|
||||
parameters = payload.get("parameters")
|
||||
if not job_type or not isinstance(parameters, dict):
|
||||
raise HTTPException(status_code=422, detail="Invalid job request")
|
||||
try:
|
||||
context = self.submit(job_type, parameters)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=422, detail="Unknown job type")
|
||||
except ValueError as error:
|
||||
raise HTTPException(status_code=409, detail=str(error))
|
||||
if context is None:
|
||||
raise HTTPException(status_code=409, detail="Service is busy")
|
||||
background_tasks.add_task(self.run, context)
|
||||
return context.snapshot()
|
||||
|
||||
@app.get("/jobs/{job_id}")
|
||||
def job_status(job_id: str):
|
||||
context = self.jobs.get(job_id)
|
||||
if context is None:
|
||||
raise HTTPException(status_code=404, detail="Job was not found")
|
||||
return context.snapshot()
|
||||
|
||||
@app.delete("/jobs/{job_id}", status_code=202)
|
||||
def cancel_job(job_id: str):
|
||||
context = self.jobs.get(job_id)
|
||||
if context is None or not context.request_cancel():
|
||||
raise HTTPException(status_code=404, detail="Active job was not found")
|
||||
return context.snapshot()
|
||||
|
||||
@app.post("/shutdown", status_code=202)
|
||||
def shutdown():
|
||||
self.shutdown_event.set()
|
||||
return {"state": "stopping"}
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def watch_parent(parent_pid, shutdown_event, interval=1.0):
|
||||
"""Stop a dedicated service after its owning Blender process exits."""
|
||||
if not parent_pid:
|
||||
return
|
||||
while not shutdown_event.wait(interval):
|
||||
if not _process_exists(parent_pid):
|
||||
shutdown_event.set()
|
||||
return
|
||||
|
||||
|
||||
def _process_exists(process_id):
|
||||
if os.name != "nt":
|
||||
try:
|
||||
os.kill(process_id, 0)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
import ctypes
|
||||
|
||||
synchronize = 0x00100000
|
||||
handle = ctypes.windll.kernel32.OpenProcess(synchronize, False, process_id)
|
||||
if not handle:
|
||||
return False
|
||||
try:
|
||||
return ctypes.windll.kernel32.WaitForSingleObject(handle, 0) == 0x00000102
|
||||
finally:
|
||||
ctypes.windll.kernel32.CloseHandle(handle)
|
||||
208
src/snapform/blendjob/service.py
Normal file
208
src/snapform/blendjob/service.py
Normal file
@@ -0,0 +1,208 @@
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .client import JobClient
|
||||
|
||||
|
||||
@dataclass
|
||||
class ServiceConnection:
|
||||
port: int
|
||||
pid: int
|
||||
instance_id: str
|
||||
client: JobClient
|
||||
|
||||
|
||||
class ServiceController:
|
||||
"""Own a dedicated local JobServer process and its HTTP connection."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
command_factory,
|
||||
*,
|
||||
cwd_factory,
|
||||
log_path_factory,
|
||||
host="127.0.0.1",
|
||||
creationflags=0,
|
||||
):
|
||||
self.command_factory = command_factory
|
||||
self.cwd_factory = cwd_factory
|
||||
self.log_path_factory = log_path_factory
|
||||
self.host = host
|
||||
self.creationflags = creationflags
|
||||
self.connection = None
|
||||
self.process = None
|
||||
self.log_file = None
|
||||
self.auto_start = False
|
||||
self.snapshot = {"state": "STOPPED", "message": "Stopped"}
|
||||
|
||||
def start(self):
|
||||
if self.process is not None and self.process.poll() is None:
|
||||
return self.connection
|
||||
self._close_process()
|
||||
port = self._available_port()
|
||||
instance_id = uuid.uuid4().hex
|
||||
log_path = self.log_path_factory()
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.log_file = log_path.open("a", encoding="utf-8")
|
||||
command = self.command_factory(port, instance_id)
|
||||
try:
|
||||
self.process = subprocess.Popen(
|
||||
command,
|
||||
stdout=self.log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
cwd=self.cwd_factory(),
|
||||
creationflags=self.creationflags,
|
||||
)
|
||||
except OSError:
|
||||
self.log_file.close()
|
||||
self.log_file = None
|
||||
raise
|
||||
client = JobClient(self.host, port)
|
||||
self.connection = ServiceConnection(
|
||||
port,
|
||||
self.process.pid,
|
||||
instance_id,
|
||||
client,
|
||||
)
|
||||
self.snapshot = {"state": "STARTING", "message": "Starting", "port": port}
|
||||
return self.connection
|
||||
|
||||
def health(self, timeout=0.3):
|
||||
if self.connection is None:
|
||||
raise RuntimeError("Service is not running")
|
||||
health = self.connection.client.request("GET", "/health", timeout=timeout)
|
||||
if health.get("instance_id") != self.connection.instance_id:
|
||||
raise RuntimeError("Service instance does not match")
|
||||
state = "BUSY" if health.get("busy") else "READY"
|
||||
self.snapshot = {
|
||||
**health,
|
||||
"state": state,
|
||||
"message": "Busy" if state == "BUSY" else "Ready",
|
||||
"port": self.connection.port,
|
||||
}
|
||||
return health
|
||||
|
||||
def ensure(self, timeout=60.0):
|
||||
self.auto_start = True
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
self.health(timeout=0.5)
|
||||
return self.connection
|
||||
except RuntimeError:
|
||||
self._close_finished_process()
|
||||
if self.process is None:
|
||||
self.start()
|
||||
time.sleep(0.05)
|
||||
raise RuntimeError(f"Service did not become ready; log: {self.log_path_factory()}")
|
||||
|
||||
def enable(self):
|
||||
self.auto_start = True
|
||||
try:
|
||||
self.health(timeout=0.2)
|
||||
except RuntimeError:
|
||||
self.start()
|
||||
return self.connection
|
||||
|
||||
def stop(self, timeout=10.0):
|
||||
self.auto_start = False
|
||||
if self.connection is not None:
|
||||
try:
|
||||
self.connection.client.shutdown()
|
||||
except RuntimeError:
|
||||
pass
|
||||
deadline = time.monotonic() + timeout
|
||||
while self.process is not None and self.process.poll() is None:
|
||||
if time.monotonic() >= deadline:
|
||||
self._close_process(terminate=True)
|
||||
break
|
||||
time.sleep(0.05)
|
||||
self._close_process()
|
||||
self.connection = None
|
||||
self.snapshot = {"state": "STOPPED", "message": "Stopped"}
|
||||
|
||||
def restart(self):
|
||||
self.stop()
|
||||
self.auto_start = True
|
||||
return self.start()
|
||||
|
||||
def submit(self, request):
|
||||
return self.ensure().client.submit_request(request)
|
||||
|
||||
def status(self, job_id):
|
||||
return self.ensure().client.status(job_id)
|
||||
|
||||
def cancel(self, job_id):
|
||||
return self.ensure().client.cancel(job_id)
|
||||
|
||||
def resource(self, name):
|
||||
return self.ensure().client.resource(name)
|
||||
|
||||
def clear_resource(self, name):
|
||||
return self.ensure().client.clear_resource(name)
|
||||
|
||||
def detach(self):
|
||||
self.auto_start = False
|
||||
self.connection = None
|
||||
self._close_process(terminate=True)
|
||||
self.snapshot = {"state": "STOPPED", "message": "Stopped"}
|
||||
|
||||
def poll(self, *, available=True):
|
||||
if not available:
|
||||
self.snapshot = {"state": "UNAVAILABLE", "message": "Environment is not installed"}
|
||||
return 2.0
|
||||
try:
|
||||
self.health(timeout=0.5)
|
||||
return 5.0
|
||||
except RuntimeError as error:
|
||||
self._close_finished_process()
|
||||
if not self.auto_start:
|
||||
self.snapshot = {"state": "STOPPED", "message": "Stopped"}
|
||||
return 2.0
|
||||
try:
|
||||
self.start()
|
||||
except OSError as start_error:
|
||||
self.snapshot = {"state": "ERROR", "message": str(start_error)}
|
||||
return 2.0
|
||||
self.snapshot = {"state": "STARTING", "message": "Starting"}
|
||||
return 1.0
|
||||
|
||||
def mark_job_complete(self, job_id):
|
||||
active = self.snapshot.get("active_job")
|
||||
if active and active.get("job_id") != job_id:
|
||||
return
|
||||
if self.snapshot.get("state") == "BUSY":
|
||||
self.snapshot = {
|
||||
**self.snapshot,
|
||||
"busy": False,
|
||||
"active_job": None,
|
||||
"state": "READY",
|
||||
"message": "Ready",
|
||||
}
|
||||
|
||||
def _available_port(self):
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
|
||||
listener.bind((self.host, 0))
|
||||
return listener.getsockname()[1]
|
||||
|
||||
def _close_finished_process(self):
|
||||
if self.process is not None and self.process.poll() is not None:
|
||||
self._close_process()
|
||||
self.connection = None
|
||||
|
||||
def _close_process(self, terminate=False):
|
||||
process = self.process
|
||||
self.process = None
|
||||
if process is not None and terminate and process.poll() is None:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=3.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=3.0)
|
||||
if self.log_file is not None:
|
||||
self.log_file.close()
|
||||
self.log_file = None
|
||||
Reference in New Issue
Block a user