LEDMatrix/src/display_watchdog.py

"""Render-loop liveness: systemd watchdog pings and a heartbeat file.

A panel can freeze while ``ledmatrix.service`` stays "active": a plugin's
``display()`` that never returns, a deadlock, a stuck hardware swap. Nothing
outside the process could tell, so nothing restarted it. This module lets the
render loop prove it is still going round, in two ways:

* **systemd watchdog.** The unit sets ``WatchdogSec=`` and
  ``NotifyAccess=main``; this sends ``WATCHDOG=1`` over ``$NOTIFY_SOCKET``.
  When the pings stop, systemd kills the process (SIGABRT, so faulthandler
  prints every thread's stack to the journal first) and ``Restart=`` brings it
  back.
* **Heartbeat file**, ``/run/ledmatrix/display-heartbeat.json``, for the web
  interface's ``/api/v3/health`` and the automatic update's health check.
  ``/run`` is tmpfs, so the writes never reach the SD card.

Both are driven only from the render thread -- ``beat()`` from any other
thread is ignored -- so a render thread stuck inside a plugin stops them even
while every other thread carries on. The unit's ``WatchdogSec=`` is the
steady-state limit; start-up (plugin loads, the 20s initial update budget,
dependency installs) is far longer and happens before the render loop exists,
so ``begin_startup()`` widens the limit for it and the render loop narrows it
back, sends ``READY=1`` and starts pinging once its first frame is on the
panel. See docs/ARCHITECTURE.md ("Liveness") for the unit settings.

Standard library only, and no import of the rest of ``src``: ``run.py`` loads
this before anything heavy so the start-up allowance is in place long before
the unit's own ``WatchdogSec`` could expire. Without ``$NOTIFY_SOCKET`` (dev
server, emulator, Windows, an older unit) every call is a cheap no-op, and the
heartbeat is written only where ``/run/ledmatrix`` exists or can be created.
"""
import contextlib
import json
import logging
import os
import socket
import tempfile
import threading
import time
from typing import Any, Callable, Dict, Iterator, Mapping, Optional

logger = logging.getLogger(__name__)

#: Where the display writes its heartbeat. ``RuntimeDirectory=ledmatrix`` in
#: the unit creates the directory; a display running under an older unit
#: creates it itself (it runs as root). The web interface, which is not root,
#: only reads it: the directory is 0755 and the file 0644.
HEARTBEAT_DIR = '/run/ledmatrix'
HEARTBEAT_NAME = 'display-heartbeat.json'
HEARTBEAT_PATH = HEARTBEAT_DIR + '/' + HEARTBEAT_NAME

#: How often the render loop pings systemd and rewrites the heartbeat. Beats
#: come many times a second; this is the rate limit on the side effects.
BEAT_INTERVAL_SECONDS = 5.0

#: A heartbeat older than this means the render loop has stopped. Above the
#: longest gap a healthy loop has (the executor's 30s display() timeout), so a
#: slow plugin does not read as a frozen panel.
HEARTBEAT_STALE_SECONDS = 60.0

#: The watchdog limit while the process starts, before the render loop runs.
#: Start-up loads every plugin (pip included, when a dependency is missing:
#: up to 300s a try), then spends up to 20s on initial updates. A hang in
#: there is still caught, just later.
STARTUP_ALLOWANCE_SECONDS = 15 * 60

#: The watchdog limit while the render thread loads a plugin that was just
#: enabled from the web UI: loading can run pip, on this thread.
PLUGIN_LOAD_ALLOWANCE_SECONDS = 15 * 60


# -- sd_notify -------------------------------------------------------------

def notify(message: str, environ: Optional[Mapping[str, str]] = None,
           socket_factory: Optional[Callable[..., Any]] = None) -> bool:
    """Send ``message`` to systemd over ``$NOTIFY_SOCKET``; True if it was sent.

    The same protocol as libsystemd's ``sd_notify()``: one datagram of
    newline-separated ``KEY=VALUE`` lines to an AF_UNIX socket. An address
    starting with ``@`` is in the abstract namespace (a leading NUL byte).
    Never raises: a missing socket or a failed send is simply False, so a
    display run outside systemd behaves exactly as before.
    """
    env = os.environ if environ is None else environ
    address = env.get('NOTIFY_SOCKET') or ''
    if address.startswith('@'):
        address = '\0' + address[1:]
    elif not address.startswith('/'):
        # Unset, or a vsock: address (systemd 253+, VMs only).
        return False
    family = getattr(socket, 'AF_UNIX', None)
    if family is None:
        return False
    factory = socket_factory or socket.socket
    try:
        sock = factory(family, socket.SOCK_DGRAM | getattr(socket, 'SOCK_CLOEXEC', 0))
        try:
            sock.connect(address)
            sock.sendall(message.encode('utf-8'))
        finally:
            sock.close()
        return True
    except OSError as e:
        logger.debug("sd_notify(%r) failed: %s", message, e)
        return False


def watchdog_usec(environ: Optional[Mapping[str, str]] = None) -> Optional[int]:
    """The unit's ``WatchdogSec`` in microseconds, or None when it has none.

    systemd passes it as ``$WATCHDOG_USEC``, with ``$WATCHDOG_PID`` naming the
    process it is meant for (a child that inherited the environment must not
    think the watchdog is its own).
    """
    env = os.environ if environ is None else environ
    pid = env.get('WATCHDOG_PID')
    if pid and pid != str(os.getpid()):
        return None
    try:
        usec = int(env.get('WATCHDOG_USEC', ''))
    except ValueError:
        return None
    return usec if usec > 0 else None


# -- heartbeat reading (web interface) ---------------------------------------

def read_heartbeat(path: str = HEARTBEAT_PATH) -> Optional[Dict[str, Any]]:
    """The heartbeat the display last wrote, or None when there is none.

    None covers a display that does not write one -- dev server, emulator,
    Windows, a display that has not drawn its first frame yet -- as well as an
    unreadable file, so callers fall back to whatever they did before.
    """
    try:
        with open(path, 'r', encoding='utf-8') as f:
            data = json.load(f)
    except (OSError, ValueError):
        return None
    return data if isinstance(data, dict) else None


def heartbeat_age(data: Mapping[str, Any], now_mono: Optional[float] = None,
                  now_wall: Optional[float] = None) -> Optional[float]:
    """Seconds since the heartbeat in ``data`` was written, or None if it has no time.

    Measured on the monotonic clock when it can be: on Linux that is
    CLOCK_MONOTONIC, shared by every process, and it does not jump when NTP
    first corrects the clock of a Pi with no RTC. /run is emptied at boot, so
    a heartbeat always comes from this boot. Falls back to the wall clock.
    """
    now_mono = time.monotonic() if now_mono is None else now_mono
    now_wall = time.time() if now_wall is None else now_wall
    mono = data.get('mono')
    if isinstance(mono, (int, float)) and not isinstance(mono, bool):
        age = now_mono - mono
        if age >= -1.0:  # a clock this far behind is not the same clock
            return max(age, 0.0)
    wall = data.get('wall')
    if isinstance(wall, (int, float)) and not isinstance(wall, bool):
        return max(now_wall - wall, 0.0)
    return None


# -- the render loop's side ----------------------------------------------------

_DEFAULT_DIR = object()


class RenderWatchdog:
    """Pings systemd and writes the heartbeat, from the render thread only.

    Lifecycle: ``begin_startup()`` as the process starts, ``bind_render_thread()``
    when ``DisplayController.run()`` starts, then ``note_frame()`` for every
    frame pushed to the panel and ``beat()`` / ``loop_pass()`` from every place
    the render loop reliably comes back to. The first beat after the first
    frame (or after the loop's first full pass, when there is nothing to draw)
    arms it: ``READY=1``, the unit's own ``WatchdogSec``, and the heartbeat.
    """

    def __init__(self, environ: Optional[Mapping[str, str]] = None,
                 send: Optional[Callable[[str], bool]] = None,
                 clock: Callable[[], float] = time.monotonic,
                 wall_clock: Callable[[], float] = time.time,
                 heartbeat_dir: Any = _DEFAULT_DIR,
                 enable_faulthandler: bool = True):
        env = dict(os.environ if environ is None else environ)
        self._send = send or (lambda message: notify(message, env))
        self._clock = clock
        self._wall_clock = wall_clock
        self._usec = watchdog_usec(env)
        if heartbeat_dir is _DEFAULT_DIR:
            # /run exists only on Linux; elsewhere (Windows dev) there is no
            # heartbeat rather than a C:\run folder.
            heartbeat_dir = HEARTBEAT_DIR if os.name == 'posix' else None
        self._heartbeat_dir: Optional[str] = heartbeat_dir
        # None until the first write; False for good if that one failed
        # (nowhere to write: not root, no /run); True once one landed.
        self._heartbeat_ok: Optional[bool] = None
        self._heartbeat_warned = False
        self._enable_faulthandler = enable_faulthandler
        self._render_thread: Optional[int] = None
        self._frame_pushed = False
        self._passes = 0
        self._armed = False
        self._last_beat: Optional[float] = None
        self._extend_depth = 0
        interval = BEAT_INTERVAL_SECONDS
        if self._usec:
            # systemd's advice is to ping at half the limit; a third leaves
            # room for one late beat even if someone sets a very short one.
            interval = min(interval, self._usec / 1e6 / 3)
        self._interval = interval

    @property
    def armed(self) -> bool:
        return self._armed

    def _on_render_thread(self) -> bool:
        return self._render_thread is not None and threading.get_ident() == self._render_thread

    def begin_startup(self) -> None:
        """Widen the watchdog to cover start-up. Call as early as possible.

        systemd starts the watchdog clock when a Type=simple service starts,
        and start-up routinely takes longer than the render loop's limit.
        Only widens: an operator who set a longer ``WatchdogSec`` keeps it.
        """
        if not self._usec:
            return
        allowance = max(self._usec, int(STARTUP_ALLOWANCE_SECONDS * 1e6))
        self._send(f'WATCHDOG_USEC={allowance}\nSTATUS=Starting: loading plugins')

    def bind_render_thread(self) -> None:
        """Mark the calling thread as the render thread; beats from others are ignored."""
        self._render_thread = threading.get_ident()
        self._frame_pushed = False
        self._passes = 0

    def note_frame(self) -> None:
        """A frame was pushed to the panel (DisplayManager.update_display).

        Any thread may push the first one -- the first dispatch of a screen
        runs on PluginExecutor's thread -- so this only records it; the
        render thread's next beat arms the watchdog.
        """
        if self._render_thread is None:
            return  # start-up screens, before the render loop exists
        self._frame_pushed = True
        if self._on_render_thread():
            self.beat()

    def loop_pass(self) -> None:
        """The top of the render loop's ``while True``.

        A second arrival here means a whole pass finished. That counts as the
        first frame when there was nothing to draw (no plugins enabled, every
        screen empty): the loop is plainly alive, and a watchdog that never
        armed would leave a later hang uncaught.
        """
        if not self._on_render_thread():
            return
        self._passes += 1
        if self._passes > 1:
            self._frame_pushed = True
        self.beat()

    def beat(self) -> None:
        """The render loop is still going round. Cheap; call it freely."""
        if not self._on_render_thread():
            return
        if not self._armed:
            if not self._frame_pushed:
                return
            self._arm()
            return
        now = self._clock()
        if self._last_beat is not None and now - self._last_beat < self._interval:
            return
        self._last_beat = now
        if self._usec:
            self._send('WATCHDOG=1')
        self._write_heartbeat(now)

    def _arm(self) -> None:
        self._armed = True
        self._last_beat = self._clock()
        if self._usec:
            # Back from the start-up allowance to the unit's own limit.
            self._send(f'READY=1\nWATCHDOG_USEC={self._usec}\nWATCHDOG=1\nSTATUS=Rendering')
            self._install_faulthandler()
            logger.info("systemd watchdog armed: the render loop must check in every %.0fs",
                        self._usec / 1e6)
        else:
            self._send('READY=1\nSTATUS=Rendering')
        self._write_heartbeat(self._last_beat)

    def _install_faulthandler(self) -> None:
        """Dump every thread's stack when the watchdog's SIGABRT arrives.

        That trace, in the journal, is what says which plugin the render
        thread was stuck in.
        """
        if not self._enable_faulthandler:
            return
        try:
            import faulthandler
            import sys
            if not faulthandler.is_enabled() and sys.stderr is not None:
                faulthandler.enable(all_threads=True)
        except (ImportError, RuntimeError, ValueError, OSError, AttributeError) as e:
            logger.debug("faulthandler not enabled: %s", e)

    @contextlib.contextmanager
    def extended(self, seconds: float, reason: str = '') -> Iterator[None]:
        """Allow the render thread ``seconds`` for one blocking job.

        For the few legitimate jobs that can outlast the watchdog, such as
        loading a newly enabled plugin, which can run pip on this thread.
        Nests; the unit's limit comes back when the outermost one ends.
        """
        if not (self._armed and self._usec and self._on_render_thread()):
            yield
            return
        usec = max(self._usec, int(seconds * 1e6))
        if self._extend_depth == 0:
            self._send(f'WATCHDOG_USEC={usec}\nWATCHDOG=1'
                       + (f'\nSTATUS=Busy: {reason}' if reason else ''))
        self._extend_depth += 1
        try:
            yield
        finally:
            self._extend_depth -= 1
            if self._extend_depth == 0:
                self._send(f'WATCHDOG_USEC={self._usec}\nWATCHDOG=1\nSTATUS=Rendering')
                self._last_beat = self._clock()
                self._write_heartbeat(self._last_beat)

    def stopping(self) -> None:
        """Clean shutdown: tell systemd, and take the heartbeat down with us.

        A heartbeat left behind by a stopped display would read as a frozen
        one to the web interface.
        """
        if self._usec or self._armed:
            self._send('STOPPING=1')
        path = self._heartbeat_path()
        if path and self._heartbeat_ok:
            try:
                os.unlink(path)
            except OSError:
                pass

    # -- heartbeat file ----------------------------------------------------

    def _heartbeat_path(self) -> Optional[str]:
        if not self._heartbeat_dir:
            return None
        return os.path.join(self._heartbeat_dir, HEARTBEAT_NAME)

    def _write_heartbeat(self, now_mono: float) -> None:
        path = self._heartbeat_path()
        if path is None or self._heartbeat_ok is False:
            return
        directory = self._heartbeat_dir
        try:
            if not os.path.isdir(directory):
                # An install whose unit predates RuntimeDirectory=: the
                # display runs as root and can make it. Anyone else cannot,
                # and gets no heartbeat -- which readers treat as "unknown".
                os.makedirs(directory, mode=0o755, exist_ok=True)
            payload = json.dumps({'pid': os.getpid(), 'mono': now_mono,
                                  'wall': self._wall_clock()})
            fd, tmp = tempfile.mkstemp(dir=directory, prefix='.heartbeat-')
            try:
                with os.fdopen(fd, 'w', encoding='utf-8') as f:
                    f.write(payload)
                os.chmod(tmp, 0o644)
                os.replace(tmp, path)
            except BaseException:
                try:
                    os.unlink(tmp)
                except OSError:
                    pass
                raise
            if self._heartbeat_ok is None:
                logger.info("Writing the display heartbeat to %s", path)
            self._heartbeat_ok = True
        except OSError as e:
            if self._heartbeat_ok is None:
                logger.info("Not writing a display heartbeat (%s: %s); health checks "
                            "fall back to their older signals", directory, e)
                self._heartbeat_ok = False
            elif not self._heartbeat_warned:
                # It worked before, so keep trying, but say so only once.
                logger.warning("Could not update the display heartbeat: %s", e)
                self._heartbeat_warned = True


#: The process-wide instance: one display process, one render loop.
watchdog = RenderWatchdog()


def beat() -> None:
    """Module-level shortcut so the Vegas loop and the plugin manager need no reference."""
    watchdog.beat()


def note_frame() -> None:
    watchdog.note_frame()


def extended(seconds: float, reason: str = ''):
    return watchdog.extended(seconds, reason)