feat(display): systemd watchdog and heartbeat for a frozen render loop (#687)

If the render loop gets stuck inside a plugin's display(), ledmatrix.service
stays active and the panel stays frozen. This adds a way to detect that.

- src/display_watchdog.py (standard library only) sends sd_notify over
  $NOTIFY_SOCKET and writes /run/ledmatrix/display-heartbeat.json. Only the
  render thread counts: beats from other threads are ignored.
- ledmatrix.service: WatchdogSec=120, NotifyAccess=main,
  RuntimeDirectory=ledmatrix (0755), RestartSteps=4 and
  RestartMaxDelaySec=2min. It stays Type=simple. run.py widens the watchdog
  to 15 min for start-up, and load_plugin() does the same on the render
  thread. The loop arms after its first frame.
- /api/v3/health adds checks.display_loop: running, stalled (no heartbeat
  for over 60s, which makes the status degraded) or not_reported. With web
  login on, a caller who is not logged in still gets only healthy/degraded,
  and a stall degrades that answer.
- The update verifier requires a fresh heartbeat from the restarted display
  when the display it replaced was writing one. A frozen panel is rolled
  back.
- Existing installs get the systemd watchdog only after install_service.sh
  is re-run. The heartbeat works right away.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/display_controller.py

@@ -34,6 +34,7 @@ from datetime import datetime
 from concurrent.futures import ThreadPoolExecutor, as_completed  # pylint: disable=no-name-in-module
 import pytz
 
+from src import display_watchdog
 from src.display_manager import DisplayManager
 from src.config_manager import ConfigManager
 from src.config_service import ConfigService
@@ -1052,6 +1053,8 @@ class DisplayController:
             the plugin's update() holds its lock (the panel keeps the last
             frame; that is not a failure).
         """
+        # Every frame of both per-screen render loops comes through here.
+        display_watchdog.watchdog.beat()
         plugin_id = getattr(plugin, 'plugin_id', None)
         with self._display_lock_or_skip(plugin_id) as can_display:
             if not can_display:
@@ -1148,6 +1151,9 @@ class DisplayController:
 
             sleep_time = min(tick_interval, remaining)
             time.sleep(sleep_time)
+            # A dwell can be a minute long (sixty seconds while scheduled
+            # off); the watchdog must hear from this thread throughout.
+            display_watchdog.watchdog.beat()
             self._tick_plugin_updates()
             self._service_pending_changes()
             if (self.current_display_mode != mode
@@ -2238,6 +2244,11 @@ class DisplayController:
                 "plugin is enabled via the web UI."
             )
 
+        # This thread is the one the systemd watchdog and the heartbeat
+        # vouch for: beats from any other thread are ignored, so a render
+        # thread stuck inside a plugin stops them.
+        display_watchdog.watchdog.bind_render_thread()
+
         try:
             # Initialize with cached data for fast startup - let background updates refresh naturally
             logger.info("Starting display with cached data (fast startup mode)")
@@ -2246,6 +2257,11 @@ class DisplayController:
             self._publish_current_mode_state()
             
             while True:
+                # Arms the watchdog after the first frame -- or after the
+                # first full pass, when there is nothing to draw -- and pings
+                # it from then on.
+                display_watchdog.watchdog.loop_pass()
+
                 # Apply plugin enable/disable edits saved via the web UI. The
                 # config-watcher thread only sets the flag; loading/unloading and
                 # rebuilding available_modes happens here on the render thread so
@@ -3625,6 +3641,9 @@ class DisplayController:
 
     def cleanup(self):
         """Clean up resources."""
+        # First: a clean stop is not a hang, and a heartbeat left behind
+        # would read as a frozen panel to the web interface.
+        display_watchdog.watchdog.stopping()
         # Stop the async update worker first so no in-flight update() call
         # is still touching display/cache-backed resources while they're
         # torn down below.

src/display_manager.py

@@ -57,6 +57,7 @@ import zlib
 import freetype
 
 from src.common import snapshot_policy
+from src import display_watchdog
 from src.common.frame_timing import FrameTimingRecorder
 
 if TYPE_CHECKING:
@@ -928,6 +929,9 @@ class DisplayManager:
                 # the fallback branch, so captured content never reaches the
                 # web preview either.
                 return
+            # The render loop's watchdog arms on the first frame to reach
+            # the panel (or the emulator/fallback path standing in for it).
+            display_watchdog.note_frame()
             with self._update_lock:
                 if self.matrix is None:
                     # Fallback mode - no actual hardware to update

src/display_watchdog.py

@@ -0,0 +1,414 @@
+"""Render-loop liveness: systemd watchdog pings and a heartbeat file.
+
+A panel can freeze while ``ledmatrix.service`` stays "active": a plugin's
+``display()`` that never returns, a deadlock, a stuck hardware swap. Nothing
+outside the process could tell, so nothing restarted it. This module lets the
+render loop prove it is still going round, in two ways:
+
+* **systemd watchdog.** The unit sets ``WatchdogSec=`` and
+  ``NotifyAccess=main``; this sends ``WATCHDOG=1`` over ``$NOTIFY_SOCKET``.
+  When the pings stop, systemd kills the process (SIGABRT, so faulthandler
+  prints every thread's stack to the journal first) and ``Restart=`` brings it
+  back.
+* **Heartbeat file**, ``/run/ledmatrix/display-heartbeat.json``, for the web
+  interface's ``/api/v3/health`` and the automatic update's health check.
+  ``/run`` is tmpfs, so the writes never reach the SD card.
+
+Both are driven only from the render thread -- ``beat()`` from any other
+thread is ignored -- so a render thread stuck inside a plugin stops them even
+while every other thread carries on. The unit's ``WatchdogSec=`` is the
+steady-state limit; start-up (plugin loads, the 20s initial update budget,
+dependency installs) is far longer and happens before the render loop exists,
+so ``begin_startup()`` widens the limit for it and the render loop narrows it
+back, sends ``READY=1`` and starts pinging once its first frame is on the
+panel. See docs/ARCHITECTURE.md ("Liveness") for the unit settings.
+
+Standard library only, and no import of the rest of ``src``: ``run.py`` loads
+this before anything heavy so the start-up allowance is in place long before
+the unit's own ``WatchdogSec`` could expire. Without ``$NOTIFY_SOCKET`` (dev
+server, emulator, Windows, an older unit) every call is a cheap no-op, and the
+heartbeat is written only where ``/run/ledmatrix`` exists or can be created.
+"""
+import contextlib
+import json
+import logging
+import os
+import socket
+import tempfile
+import threading
+import time
+from typing import Any, Callable, Dict, Iterator, Mapping, Optional
+
+logger = logging.getLogger(__name__)
+
+#: Where the display writes its heartbeat. ``RuntimeDirectory=ledmatrix`` in
+#: the unit creates the directory; a display running under an older unit
+#: creates it itself (it runs as root). The web interface, which is not root,
+#: only reads it: the directory is 0755 and the file 0644.
+HEARTBEAT_DIR = '/run/ledmatrix'
+HEARTBEAT_NAME = 'display-heartbeat.json'
+HEARTBEAT_PATH = HEARTBEAT_DIR + '/' + HEARTBEAT_NAME
+
+#: How often the render loop pings systemd and rewrites the heartbeat. Beats
+#: come many times a second; this is the rate limit on the side effects.
+BEAT_INTERVAL_SECONDS = 5.0
+
+#: A heartbeat older than this means the render loop has stopped. Above the
+#: longest gap a healthy loop has (the executor's 30s display() timeout), so a
+#: slow plugin does not read as a frozen panel.
+HEARTBEAT_STALE_SECONDS = 60.0
+
+#: The watchdog limit while the process starts, before the render loop runs.
+#: Start-up loads every plugin (pip included, when a dependency is missing:
+#: up to 300s a try), then spends up to 20s on initial updates. A hang in
+#: there is still caught, just later.
+STARTUP_ALLOWANCE_SECONDS = 15 * 60
+
+#: The watchdog limit while the render thread loads a plugin that was just
+#: enabled from the web UI: loading can run pip, on this thread.
+PLUGIN_LOAD_ALLOWANCE_SECONDS = 15 * 60
+
+
+# -- sd_notify -------------------------------------------------------------
+
+def notify(message: str, environ: Optional[Mapping[str, str]] = None,
+           socket_factory: Optional[Callable[..., Any]] = None) -> bool:
+    """Send ``message`` to systemd over ``$NOTIFY_SOCKET``; True if it was sent.
+
+    The same protocol as libsystemd's ``sd_notify()``: one datagram of
+    newline-separated ``KEY=VALUE`` lines to an AF_UNIX socket. An address
+    starting with ``@`` is in the abstract namespace (a leading NUL byte).
+    Never raises: a missing socket or a failed send is simply False, so a
+    display run outside systemd behaves exactly as before.
+    """
+    env = os.environ if environ is None else environ
+    address = env.get('NOTIFY_SOCKET') or ''
+    if address.startswith('@'):
+        address = '\0' + address[1:]
+    elif not address.startswith('/'):
+        # Unset, or a vsock: address (systemd 253+, VMs only).
+        return False
+    family = getattr(socket, 'AF_UNIX', None)
+    if family is None:
+        return False
+    factory = socket_factory or socket.socket
+    try:
+        sock = factory(family, socket.SOCK_DGRAM | getattr(socket, 'SOCK_CLOEXEC', 0))
+        try:
+            sock.connect(address)
+            sock.sendall(message.encode('utf-8'))
+        finally:
+            sock.close()
+        return True
+    except OSError as e:
+        logger.debug("sd_notify(%r) failed: %s", message, e)
+        return False
+
+
+def watchdog_usec(environ: Optional[Mapping[str, str]] = None) -> Optional[int]:
+    """The unit's ``WatchdogSec`` in microseconds, or None when it has none.
+
+    systemd passes it as ``$WATCHDOG_USEC``, with ``$WATCHDOG_PID`` naming the
+    process it is meant for (a child that inherited the environment must not
+    think the watchdog is its own).
+    """
+    env = os.environ if environ is None else environ
+    pid = env.get('WATCHDOG_PID')
+    if pid and pid != str(os.getpid()):
+        return None
+    try:
+        usec = int(env.get('WATCHDOG_USEC', ''))
+    except ValueError:
+        return None
+    return usec if usec > 0 else None
+
+
+# -- heartbeat reading (web interface) ---------------------------------------
+
+def read_heartbeat(path: str = HEARTBEAT_PATH) -> Optional[Dict[str, Any]]:
+    """The heartbeat the display last wrote, or None when there is none.
+
+    None covers a display that does not write one -- dev server, emulator,
+    Windows, a display that has not drawn its first frame yet -- as well as an
+    unreadable file, so callers fall back to whatever they did before.
+    """
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+    except (OSError, ValueError):
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def heartbeat_age(data: Mapping[str, Any], now_mono: Optional[float] = None,
+                  now_wall: Optional[float] = None) -> Optional[float]:
+    """Seconds since the heartbeat in ``data`` was written, or None if it has no time.
+
+    Measured on the monotonic clock when it can be: on Linux that is
+    CLOCK_MONOTONIC, shared by every process, and it does not jump when NTP
+    first corrects the clock of a Pi with no RTC. /run is emptied at boot, so
+    a heartbeat always comes from this boot. Falls back to the wall clock.
+    """
+    now_mono = time.monotonic() if now_mono is None else now_mono
+    now_wall = time.time() if now_wall is None else now_wall
+    mono = data.get('mono')
+    if isinstance(mono, (int, float)) and not isinstance(mono, bool):
+        age = now_mono - mono
+        if age >= -1.0:  # a clock this far behind is not the same clock
+            return max(age, 0.0)
+    wall = data.get('wall')
+    if isinstance(wall, (int, float)) and not isinstance(wall, bool):
+        return max(now_wall - wall, 0.0)
+    return None
+
+
+# -- the render loop's side ----------------------------------------------------
+
+_DEFAULT_DIR = object()
+
+
+class RenderWatchdog:
+    """Pings systemd and writes the heartbeat, from the render thread only.
+
+    Lifecycle: ``begin_startup()`` as the process starts, ``bind_render_thread()``
+    when ``DisplayController.run()`` starts, then ``note_frame()`` for every
+    frame pushed to the panel and ``beat()`` / ``loop_pass()`` from every place
+    the render loop reliably comes back to. The first beat after the first
+    frame (or after the loop's first full pass, when there is nothing to draw)
+    arms it: ``READY=1``, the unit's own ``WatchdogSec``, and the heartbeat.
+    """
+
+    def __init__(self, environ: Optional[Mapping[str, str]] = None,
+                 send: Optional[Callable[[str], bool]] = None,
+                 clock: Callable[[], float] = time.monotonic,
+                 wall_clock: Callable[[], float] = time.time,
+                 heartbeat_dir: Any = _DEFAULT_DIR,
+                 enable_faulthandler: bool = True):
+        env = dict(os.environ if environ is None else environ)
+        self._send = send or (lambda message: notify(message, env))
+        self._clock = clock
+        self._wall_clock = wall_clock
+        self._usec = watchdog_usec(env)
+        if heartbeat_dir is _DEFAULT_DIR:
+            # /run exists only on Linux; elsewhere (Windows dev) there is no
+            # heartbeat rather than a C:\run folder.
+            heartbeat_dir = HEARTBEAT_DIR if os.name == 'posix' else None
+        self._heartbeat_dir: Optional[str] = heartbeat_dir
+        # None until the first write; False for good if that one failed
+        # (nowhere to write: not root, no /run); True once one landed.
+        self._heartbeat_ok: Optional[bool] = None
+        self._heartbeat_warned = False
+        self._enable_faulthandler = enable_faulthandler
+        self._render_thread: Optional[int] = None
+        self._frame_pushed = False
+        self._passes = 0
+        self._armed = False
+        self._last_beat: Optional[float] = None
+        self._extend_depth = 0
+        interval = BEAT_INTERVAL_SECONDS
+        if self._usec:
+            # systemd's advice is to ping at half the limit; a third leaves
+            # room for one late beat even if someone sets a very short one.
+            interval = min(interval, self._usec / 1e6 / 3)
+        self._interval = interval
+
+    @property
+    def armed(self) -> bool:
+        return self._armed
+
+    def _on_render_thread(self) -> bool:
+        return self._render_thread is not None and threading.get_ident() == self._render_thread
+
+    def begin_startup(self) -> None:
+        """Widen the watchdog to cover start-up. Call as early as possible.
+
+        systemd starts the watchdog clock when a Type=simple service starts,
+        and start-up routinely takes longer than the render loop's limit.
+        Only widens: an operator who set a longer ``WatchdogSec`` keeps it.
+        """
+        if not self._usec:
+            return
+        allowance = max(self._usec, int(STARTUP_ALLOWANCE_SECONDS * 1e6))
+        self._send(f'WATCHDOG_USEC={allowance}\nSTATUS=Starting: loading plugins')
+
+    def bind_render_thread(self) -> None:
+        """Mark the calling thread as the render thread; beats from others are ignored."""
+        self._render_thread = threading.get_ident()
+        self._frame_pushed = False
+        self._passes = 0
+
+    def note_frame(self) -> None:
+        """A frame was pushed to the panel (DisplayManager.update_display).
+
+        Any thread may push the first one -- the first dispatch of a screen
+        runs on PluginExecutor's thread -- so this only records it; the
+        render thread's next beat arms the watchdog.
+        """
+        if self._render_thread is None:
+            return  # start-up screens, before the render loop exists
+        self._frame_pushed = True
+        if self._on_render_thread():
+            self.beat()
+
+    def loop_pass(self) -> None:
+        """The top of the render loop's ``while True``.
+
+        A second arrival here means a whole pass finished. That counts as the
+        first frame when there was nothing to draw (no plugins enabled, every
+        screen empty): the loop is plainly alive, and a watchdog that never
+        armed would leave a later hang uncaught.
+        """
+        if not self._on_render_thread():
+            return
+        self._passes += 1
+        if self._passes > 1:
+            self._frame_pushed = True
+        self.beat()
+
+    def beat(self) -> None:
+        """The render loop is still going round. Cheap; call it freely."""
+        if not self._on_render_thread():
+            return
+        if not self._armed:
+            if not self._frame_pushed:
+                return
+            self._arm()
+            return
+        now = self._clock()
+        if self._last_beat is not None and now - self._last_beat < self._interval:
+            return
+        self._last_beat = now
+        if self._usec:
+            self._send('WATCHDOG=1')
+        self._write_heartbeat(now)
+
+    def _arm(self) -> None:
+        self._armed = True
+        self._last_beat = self._clock()
+        if self._usec:
+            # Back from the start-up allowance to the unit's own limit.
+            self._send(f'READY=1\nWATCHDOG_USEC={self._usec}\nWATCHDOG=1\nSTATUS=Rendering')
+            self._install_faulthandler()
+            logger.info("systemd watchdog armed: the render loop must check in every %.0fs",
+                        self._usec / 1e6)
+        else:
+            self._send('READY=1\nSTATUS=Rendering')
+        self._write_heartbeat(self._last_beat)
+
+    def _install_faulthandler(self) -> None:
+        """Dump every thread's stack when the watchdog's SIGABRT arrives.
+
+        That trace, in the journal, is what says which plugin the render
+        thread was stuck in.
+        """
+        if not self._enable_faulthandler:
+            return
+        try:
+            import faulthandler
+            import sys
+            if not faulthandler.is_enabled() and sys.stderr is not None:
+                faulthandler.enable(all_threads=True)
+        except (ImportError, RuntimeError, ValueError, OSError, AttributeError) as e:
+            logger.debug("faulthandler not enabled: %s", e)
+
+    @contextlib.contextmanager
+    def extended(self, seconds: float, reason: str = '') -> Iterator[None]:
+        """Allow the render thread ``seconds`` for one blocking job.
+
+        For the few legitimate jobs that can outlast the watchdog, such as
+        loading a newly enabled plugin, which can run pip on this thread.
+        Nests; the unit's limit comes back when the outermost one ends.
+        """
+        if not (self._armed and self._usec and self._on_render_thread()):
+            yield
+            return
+        usec = max(self._usec, int(seconds * 1e6))
+        if self._extend_depth == 0:
+            self._send(f'WATCHDOG_USEC={usec}\nWATCHDOG=1'
+                       + (f'\nSTATUS=Busy: {reason}' if reason else ''))
+        self._extend_depth += 1
+        try:
+            yield
+        finally:
+            self._extend_depth -= 1
+            if self._extend_depth == 0:
+                self._send(f'WATCHDOG_USEC={self._usec}\nWATCHDOG=1\nSTATUS=Rendering')
+                self._last_beat = self._clock()
+                self._write_heartbeat(self._last_beat)
+
+    def stopping(self) -> None:
+        """Clean shutdown: tell systemd, and take the heartbeat down with us.
+
+        A heartbeat left behind by a stopped display would read as a frozen
+        one to the web interface.
+        """
+        if self._usec or self._armed:
+            self._send('STOPPING=1')
+        path = self._heartbeat_path()
+        if path and self._heartbeat_ok:
+            try:
+                os.unlink(path)
+            except OSError:
+                pass
+
+    # -- heartbeat file ----------------------------------------------------
+
+    def _heartbeat_path(self) -> Optional[str]:
+        if not self._heartbeat_dir:
+            return None
+        return os.path.join(self._heartbeat_dir, HEARTBEAT_NAME)
+
+    def _write_heartbeat(self, now_mono: float) -> None:
+        path = self._heartbeat_path()
+        if path is None or self._heartbeat_ok is False:
+            return
+        directory = self._heartbeat_dir
+        try:
+            if not os.path.isdir(directory):
+                # An install whose unit predates RuntimeDirectory=: the
+                # display runs as root and can make it. Anyone else cannot,
+                # and gets no heartbeat -- which readers treat as "unknown".
+                os.makedirs(directory, mode=0o755, exist_ok=True)
+            payload = json.dumps({'pid': os.getpid(), 'mono': now_mono,
+                                  'wall': self._wall_clock()})
+            fd, tmp = tempfile.mkstemp(dir=directory, prefix='.heartbeat-')
+            try:
+                with os.fdopen(fd, 'w', encoding='utf-8') as f:
+                    f.write(payload)
+                os.chmod(tmp, 0o644)
+                os.replace(tmp, path)
+            except BaseException:
+                try:
+                    os.unlink(tmp)
+                except OSError:
+                    pass
+                raise
+            if self._heartbeat_ok is None:
+                logger.info("Writing the display heartbeat to %s", path)
+            self._heartbeat_ok = True
+        except OSError as e:
+            if self._heartbeat_ok is None:
+                logger.info("Not writing a display heartbeat (%s: %s); health checks "
+                            "fall back to their older signals", directory, e)
+                self._heartbeat_ok = False
+            elif not self._heartbeat_warned:
+                # It worked before, so keep trying, but say so only once.
+                logger.warning("Could not update the display heartbeat: %s", e)
+                self._heartbeat_warned = True
+
+
+#: The process-wide instance: one display process, one render loop.
+watchdog = RenderWatchdog()
+
+
+def beat() -> None:
+    """Module-level shortcut so the Vegas loop and the plugin manager need no reference."""
+    watchdog.beat()
+
+
+def note_frame() -> None:
+    watchdog.note_frame()
+
+
+def extended(seconds: float, reason: str = ''):
+    return watchdog.extended(seconds, reason)

src/plugin_system/plugin_manager.py

@@ -18,6 +18,7 @@ import types
 from pathlib import Path
 from typing import Dict, List, NamedTuple, Optional, Any, Tuple, Union
 import logging
+from src import display_watchdog
 from src.exceptions import PluginError, ConfigError
 from src.logging_config import get_logger
 from src.plugin_system.plugin_loader import PluginLoader
@@ -354,6 +355,19 @@ class PluginManager:
         return plugin_ids
 
     def load_plugin(self, plugin_id: str, force_enabled: bool = False) -> bool:
+        """Load a plugin by ID; see _load_plugin.
+
+        Loading can install the plugin's dependencies with pip -- minutes,
+        not seconds. When that happens on the display's render thread (a
+        plugin enabled from the web UI, or loaded for on-demand), its
+        systemd watchdog gets a longer limit for the duration. Start-up
+        loads, on a thread pool, are covered by the start-up allowance.
+        """
+        with display_watchdog.extended(display_watchdog.PLUGIN_LOAD_ALLOWANCE_SECONDS,
+                                       f'loading plugin {plugin_id}'):
+            return self._load_plugin(plugin_id, force_enabled)
+
+    def _load_plugin(self, plugin_id: str, force_enabled: bool = False) -> bool:
         """
         Load a plugin by ID.
         
@@ -1244,6 +1258,9 @@ class PluginManager:
                 # Kill-switch path: the original inline execution
                 # (blocks the caller until update() completes/times out)
                 self._execute_update_now(plugin_id, plugin_instance, current_time)
+                # Up to the executor's 30s each, one after another on the
+                # render thread: check in with its watchdog between them.
+                display_watchdog.beat()
             else:
                 self._enqueue_update(plugin_id, current_time)
 

src/vegas_mode/coordinator.py

@@ -20,6 +20,7 @@ import time
 import threading
 from typing import Optional, Dict, Any, List, Callable, TYPE_CHECKING
 
+from src import display_watchdog
 from src.common import render_gate
 from src.vegas_mode.config import VegasModeConfig
 from src.vegas_mode.plugin_adapter import PluginAdapter
@@ -542,6 +543,9 @@ class VegasModeCoordinator:
             # the whole budget -- the render loop stalls for the size of the
             # correction. A forward jump inflates p99 and worst-frame instead.
             frame_started = time.monotonic()
+            # An iteration runs for minutes (max_cycle_duration) without
+            # returning to the display controller's loop.
+            display_watchdog.beat()
 
             # Check for STATIC mode plugin that should pause scroll
             static_plugin = self._check_static_plugin_trigger()
@@ -923,6 +927,7 @@ class VegasModeCoordinator:
 
                 # Sleep in small increments to remain responsive
                 time.sleep(0.1)
+                display_watchdog.beat()
 
             logger.info(
                 "Static pause completed for %s after %.1fs",

src/vegas_mode/stream_manager.py

@@ -21,6 +21,7 @@ from collections import deque
 from dataclasses import dataclass, field
 from PIL import Image
 
+from src import display_watchdog
 from src.vegas_mode.config import VegasModeConfig
 from src.vegas_mode.plugin_adapter import PluginAdapter
 from src.plugin_system.base_plugin import VegasDisplayMode, resolve_vegas_participation
@@ -562,6 +563,9 @@ class StreamManager:
         Returns:
             ContentSegment or None if fetch failed
         """
+        # Composing a cycle fetches plugin after plugin on the render thread
+        # (on the prefetch thread this is ignored), so check in between.
+        display_watchdog.beat()
         try:
             if not hasattr(self.plugin_manager, 'plugins'):
                 logger.warning("[%s] plugin_manager has no plugins attribute", plugin_id)