mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-10-05 14:55:08 +00:00
feat(web): a start that waits for the display answers 202 and is delivered in the background
The start route held a request open for up to 45 s while a cold-started display loaded its plugins; the MQTT bridge (15 s timeout) and browsers reported a failure for a request that was then delivered. Now, when no display is listening, the route starts the service if asked and answers 202 with status "starting" at once. A single worker in the web process (web_interface/on_demand_dispatch.py) sends the request until the display acknowledges it or the wait runs out (45 s cold start, 10 s for a running service without a socket yet). A newer start supersedes the pending one; a stop cancels it (and succeeds, with cancelled_request_id, even with no display listening). The outcome is reported by /display/on-demand/status (source "web": starting, or error with start-timeout or the socket's reason, until the display publishes something newer) and by /display/current-status as on_demand_pending. Callers: the web UI's on-demand modal and "Preview on display" treat "starting" as taken (an info toast); the MQTT bridge already treats any non-error 2xx as success (now pinned by a test). Tests: the dispatcher (ack, retry then ack, start-timeout, other failures, superseded, an in-flight ack for a superseded start, stop while pending, a per-start wait, outcome lifetime); the routes (202, status routes while pending and after a timeout, a later display state replacing the failure, stop while pending, a new start superseding); a JS suite for app.js. Mutation check: 20 mutants on the worker, the routes and app.js, 20 killed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chuck
@@ -0,0 +1,221 @@
|
||||
"""Deliver an on-demand start to a display that is not listening yet.
|
||||
|
||||
``POST /api/v3/display/on-demand/start`` can find no display on the control
|
||||
socket: the service is stopped (the route starts it) or still loading its
|
||||
plugins. The socket comes up only when the display's run loop starts, which
|
||||
can take longer than a client waits -- the MQTT bridge gives up after 15 s.
|
||||
So the route answers at once (``202``, ``status: "starting"``) and hands
|
||||
the request to the one :class:`OnDemandDispatcher` of the web process, whose
|
||||
worker thread sends it again until the display acknowledges it or
|
||||
:data:`START_WAIT_SECONDS` pass.
|
||||
|
||||
* One request at a time: a newer start replaces the pending one, and a
|
||||
stop cancels it (:meth:`OnDemandDispatcher.cancel`).
|
||||
* Its outcome is :meth:`OnDemandDispatcher.status`, which
|
||||
``GET /display/on-demand/status`` and ``/display/current-status`` report:
|
||||
``starting`` while it waits, ``delivered`` once acknowledged (the
|
||||
display's own state takes over from there), or ``error`` with
|
||||
``start-timeout`` or the socket's reason.
|
||||
|
||||
Nothing is written to disk: the file mailbox that once carried such a
|
||||
request is gone (docs/IPC_CONTROL_SOCKET.md, stage 5).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from src.ipc import client as control_client
|
||||
from src.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
#: How long the worker keeps sending a start before it gives up
|
||||
#: (``start-timeout``). The socket comes up when the display's run loop
|
||||
#: starts, after every plugin has loaded.
|
||||
START_WAIT_SECONDS = 45.0
|
||||
|
||||
#: Gap between two sends while nothing is listening.
|
||||
RETRY_INTERVAL = 0.5
|
||||
|
||||
#: How long a finished outcome (delivered, error, cancelled) is still
|
||||
#: reported, so a client polling every few seconds sees it.
|
||||
OUTCOME_SECONDS = 120.0
|
||||
|
||||
#: ``send(payload)`` hands the request to the display (the route's
|
||||
#: ``_send_on_demand``) and raises ``ControlError`` when it does not take it.
|
||||
Sender = Callable[[Dict[str, Any]], Any]
|
||||
|
||||
|
||||
class OnDemandDispatcher:
|
||||
"""One pending on-demand start, and the worker thread that delivers it."""
|
||||
|
||||
def __init__(self, send: Sender, *,
|
||||
wait_seconds: Optional[float] = None,
|
||||
retry_interval: Optional[float] = None,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
wall_clock: Callable[[], float] = time.time):
|
||||
self._send = send
|
||||
self.wait_seconds = START_WAIT_SECONDS if wait_seconds is None else wait_seconds
|
||||
self.retry_interval = RETRY_INTERVAL if retry_interval is None else retry_interval
|
||||
self._clock = clock
|
||||
self._wall = wall_clock
|
||||
self._lock = threading.Lock()
|
||||
self._wake = threading.Event()
|
||||
# Bumped by every submit and cancel: a send that started under an
|
||||
# older generation does not report its result as the current one.
|
||||
self._generation = 0
|
||||
self._pending: Optional[Dict[str, Any]] = None
|
||||
self._deadline = 0.0
|
||||
self._status: Optional[Dict[str, Any]] = None
|
||||
self._finished_at: Optional[float] = None
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
|
||||
# -- the routes' side ------------------------------------------------------
|
||||
|
||||
def submit(self, payload: Dict[str, Any], wait_seconds: Optional[float] = None) -> None:
|
||||
"""Deliver ``payload`` (an on-demand start) in the background,
|
||||
replacing any start still pending."""
|
||||
with self._lock:
|
||||
self._generation += 1
|
||||
superseded = self._pending
|
||||
self._pending = dict(payload)
|
||||
self._deadline = self._clock() + (self.wait_seconds if wait_seconds is None
|
||||
else wait_seconds)
|
||||
self._status = self._describe('starting', payload)
|
||||
self._finished_at = None
|
||||
if self._thread is None or not self._thread.is_alive():
|
||||
self._thread = threading.Thread(target=self._run, name='on-demand-dispatch',
|
||||
daemon=True)
|
||||
self._thread.start()
|
||||
if superseded is not None:
|
||||
logger.info("On-demand start %s superseded by %s before the display took it",
|
||||
superseded.get('request_id'), payload.get('request_id'))
|
||||
self._wake.set()
|
||||
|
||||
def cancel(self, reason: str = 'cancelled') -> Optional[str]:
|
||||
"""Drop the pending start (a stop arrived). Returns its request id,
|
||||
or None when nothing was pending."""
|
||||
with self._lock:
|
||||
pending = self._pending
|
||||
if pending is None:
|
||||
return None
|
||||
self._generation += 1
|
||||
self._pending = None
|
||||
self._finish(self._describe('idle', pending, last_event=reason))
|
||||
self._wake.set()
|
||||
logger.info("On-demand start %s cancelled before the display took it (%s)",
|
||||
pending.get('request_id'), reason)
|
||||
return pending.get('request_id')
|
||||
|
||||
def status(self) -> Optional[Dict[str, Any]]:
|
||||
"""The pending start's state, or its outcome for OUTCOME_SECONDS
|
||||
after it finished; None otherwise. In the shape of the display's
|
||||
on-demand state (``active``, ``status``, ``error``, ...), plus
|
||||
``source: "web"``."""
|
||||
with self._lock:
|
||||
if self._status is None:
|
||||
return None
|
||||
if (self._finished_at is not None
|
||||
and self._clock() - self._finished_at > OUTCOME_SECONDS):
|
||||
return None
|
||||
return dict(self._status)
|
||||
|
||||
def pending(self) -> bool:
|
||||
with self._lock:
|
||||
return self._pending is not None
|
||||
|
||||
# -- the worker ------------------------------------------------------------
|
||||
|
||||
def _describe(self, status: str, payload: Dict[str, Any], error: Optional[str] = None,
|
||||
last_event: Optional[str] = None) -> Dict[str, Any]:
|
||||
return {
|
||||
'active': False,
|
||||
'status': status,
|
||||
'error': error,
|
||||
'last_event': last_event,
|
||||
'request_id': payload.get('request_id'),
|
||||
'plugin_id': payload.get('plugin_id'),
|
||||
'mode': payload.get('mode'),
|
||||
'duration': payload.get('duration'),
|
||||
'pinned': bool(payload.get('pinned', False)),
|
||||
'last_updated': self._wall(),
|
||||
'source': 'web',
|
||||
}
|
||||
|
||||
def _finish(self, status: Dict[str, Any]) -> None:
|
||||
"""Record an outcome. Caller holds _lock."""
|
||||
self._status = status
|
||||
self._finished_at = self._clock()
|
||||
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
with self._lock:
|
||||
payload, generation = self._pending, self._generation
|
||||
deadline = self._deadline
|
||||
if payload is None:
|
||||
self._thread = None
|
||||
return
|
||||
outcome, error = self._attempt(payload)
|
||||
with self._lock:
|
||||
if generation != self._generation:
|
||||
continue # superseded or cancelled meanwhile
|
||||
if outcome == 'retry' and self._clock() + self.retry_interval > deadline:
|
||||
outcome, error = 'error', 'start-timeout'
|
||||
if outcome == 'delivered':
|
||||
self._pending = None
|
||||
self._finish(self._describe('delivered', payload,
|
||||
last_event='delivered'))
|
||||
elif outcome == 'error':
|
||||
self._pending = None
|
||||
self._finish(self._describe('error', payload, error=error))
|
||||
else:
|
||||
self._wake.clear()
|
||||
if outcome == 'delivered':
|
||||
logger.info("On-demand start %s delivered once the display was listening",
|
||||
payload.get('request_id'))
|
||||
elif outcome == 'error':
|
||||
logger.warning("On-demand start %s not delivered: %s",
|
||||
payload.get('request_id'), error)
|
||||
else:
|
||||
# A submit or cancel wakes the wait at once.
|
||||
self._wake.wait(self.retry_interval)
|
||||
|
||||
def _attempt(self, payload: Dict[str, Any]):
|
||||
try:
|
||||
self._send(payload)
|
||||
except control_client.ControlError as e:
|
||||
if control_client.display_not_listening(e):
|
||||
return 'retry', None
|
||||
return 'error', str(e.reason)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
logger.exception("On-demand start %s: the control socket client failed",
|
||||
payload.get('request_id'))
|
||||
return 'error', 'internal'
|
||||
return 'delivered', None
|
||||
|
||||
|
||||
_dispatcher: Optional[OnDemandDispatcher] = None
|
||||
_dispatcher_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_dispatcher(send: Sender) -> OnDemandDispatcher:
|
||||
"""The web process's dispatcher, created on first use."""
|
||||
global _dispatcher
|
||||
with _dispatcher_lock:
|
||||
if _dispatcher is None:
|
||||
_dispatcher = OnDemandDispatcher(send)
|
||||
return _dispatcher
|
||||
|
||||
|
||||
def current() -> Optional[OnDemandDispatcher]:
|
||||
"""The dispatcher if one was created, without creating one."""
|
||||
return _dispatcher
|
||||
|
||||
|
||||
def reset_for_tests() -> None:
|
||||
global _dispatcher
|
||||
with _dispatcher_lock:
|
||||
_dispatcher = None
|
||||
Reference in New Issue
Block a user