Serve the settings search index from the server as JSON

Move index building off the client so there is no client-side HTML fetching
or DOM parsing (resolves Codacy's variable-fetch and DOMParser flags on the
read-only, same-origin index build).

- Add GET /v3/settings/search-index (pages_v3.py): renders the settings
  partials server-side and extracts each field's anchor id, key, label,
  tooltip, and section with a small stdlib HTMLParser, then caches the result
  keyed on the installed-plugin set. Parsing the rendered HTML keeps anchor
  ids identical to the live DOM, so the index cannot drift.
- settings-search.js: buildIndex() now does a single fetch of the literal
  endpoint + .json(); removed the per-partial fetch loop, DOMParser, scanDoc,
  CORE_TABS, and the plugin-id allowlist. Search, keyboard nav, navigation,
  and the per-tab filter are unchanged.

Net: fewer requests and no client-side HTML parsing. Verified with a new
endpoint test in test_web_settings_ui.py (12 pass) and the headless-Chromium
test (tooltip, filter, search navigate + flash all green).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014gZxznuxw8L92FUMBN3Nqz

test/test_web_settings_ui.py

@@ -115,3 +115,28 @@ def test_display_tooltip_carries_rich_text(client):
     body = client.get("/v3/partials/display").get_data(as_text=True)
     assert 'id="setting-display-brightness"' in body
     assert "Recommended:" in body  # rich detail authored into a tooltip
+
+
+def test_search_index_endpoint(client):
+    """The server-built search index powers the global settings search."""
+    resp = client.get("/v3/settings/search-index")
+    assert resp.status_code == 200
+    data = resp.get_json()
+    assert isinstance(data, dict) and isinstance(data.get("fields"), list)
+    fields = data["fields"]
+    assert len(fields) >= 40, "expected the core settings fields to be indexed"
+
+    by_id = {f["anchorId"]: f for f in fields}
+    # Representative fields across tabs must be present with usable text.
+    for anchor in ("setting-general-timezone", "setting-display-brightness",
+                   "setting-wifi-password"):
+        assert anchor in by_id, f"{anchor} missing from search index"
+        entry = by_id[anchor]
+        assert entry["label"], f"{anchor} has no label"
+        assert entry["help"], f"{anchor} has no tooltip help"
+        assert entry["tab"] and entry["tabLabel"]
+
+    # Every entry must carry a non-empty label and a stable anchor id.
+    assert all(f["label"] and f["anchorId"].startswith("setting-") for f in fields)
+    # Section context is captured for grouped fields (e.g. Display hardware).
+    assert by_id["setting-display-brightness"]["section"] == "Hardware Configuration"

web_interface/blueprints/pages_v3.py

@@ -1,6 +1,7 @@
-from flask import Blueprint, render_template, flash
+from flask import Blueprint, render_template, flash, jsonify
 from jinja2 import TemplateNotFound
 from markupsafe import escape
+from html.parser import HTMLParser
 import json
 import logging
 import os
@@ -22,6 +23,114 @@ plugin_store_manager = None
 
 pages_v3 = Blueprint('pages_v3', __name__)
 
+
+class _SettingsIndexParser(HTMLParser):
+    """Extract searchable settings fields from a rendered partial's HTML.
+
+    Captures one entry per ``<div class="form-group" id="setting-…">``: the
+    anchor id, ``data-setting-key``, the field's ``<label>`` text, the
+    ``.help-tip`` tooltip text (``data-tooltip``), and the nearest preceding
+    ``<h3>``/``<h4>`` section heading. Parsing the *rendered* HTML (rather than
+    the schema) guarantees the anchor ids match the live DOM exactly, so the
+    search index cannot drift from what users actually see.
+    """
+
+    def __init__(self, tab, tab_label):
+        super().__init__(convert_charrefs=True)
+        self.tab = tab
+        self.tab_label = tab_label
+        self.fields = []
+        self._section = ''
+        self._field = None
+        self._depth = 0            # open-div depth within the current field
+        self._in_label = False
+        self._label_parts = []
+        self._in_heading = False
+        self._heading_parts = []
+
+    def handle_starttag(self, tag, attrs):
+        a = {k: (v or '') for k, v in attrs}
+        classes = a.get('class', '').split()
+        # Section headings (only when not already inside a field)
+        if tag in ('h3', 'h4') and self._field is None:
+            self._in_heading = True
+            self._heading_parts = []
+        if tag == 'div':
+            fid = a.get('id', '')
+            if self._field is None and 'form-group' in classes and fid.startswith('setting-'):
+                self._field = {
+                    'anchorId': fid,
+                    'key': a.get('data-setting-key', '') or fid[len('setting-'):],
+                    'label': '',
+                    'help': '',
+                    'section': self._section,
+                    'tab': self.tab,
+                    'tabLabel': self.tab_label,
+                }
+                self._depth = 1
+                return
+            if self._field is not None:
+                self._depth += 1
+        if self._field is not None:
+            if tag == 'label' and not self._field['label']:
+                self._in_label = True
+                self._label_parts = []
+            if tag == 'button' and 'help-tip' in classes and not self._field['help']:
+                self._field['help'] = a.get('data-tooltip', '')
+
+    def handle_data(self, data):
+        if self._in_label:
+            self._label_parts.append(data)
+        elif self._in_heading:
+            self._heading_parts.append(data)
+
+    def handle_endtag(self, tag):
+        if tag in ('h3', 'h4') and self._in_heading:
+            self._in_heading = False
+            self._section = ' '.join(''.join(self._heading_parts).split()).strip()
+            return
+        if self._field is None:
+            return
+        if tag == 'label' and self._in_label:
+            self._in_label = False
+            self._field['label'] = ' '.join(''.join(self._label_parts).split()).strip()
+        elif tag == 'div':
+            self._depth -= 1
+            if self._depth <= 0:
+                if self._field['label']:
+                    self.fields.append(self._field)
+                self._field = None
+                self._depth = 0
+
+
+def _partial_html(loader):
+    """Run a partial loader and return its HTML string ('' on error)."""
+    try:
+        result = loader()
+    except Exception:
+        logger.warning("search-index: partial render failed", exc_info=True)
+        return ''
+    if isinstance(result, str):
+        return result
+    if isinstance(result, tuple):  # loaders return (msg, status) on error
+        return ''
+    try:
+        return result.get_data(as_text=True)
+    except Exception:
+        return ''
+
+
+def _extract_settings_fields(html, tab, tab_label):
+    parser = _SettingsIndexParser(tab, tab_label)
+    parser.feed(html)
+    return parser.fields
+
+
+# Cache the built index keyed on the installed-plugin set. Core labels/tooltips
+# are static template text, so only a change in installed plugins invalidates it.
+_SEARCH_INDEX_CACHE = {'sig': None, 'fields': None}
+
+
 @pages_v3.route('/')
 def index():
     """Main v3 interface page"""
@@ -111,6 +220,55 @@ def load_plugin_config_partial(plugin_id):
         return '<div class="text-red-500 p-4">Error loading plugin config; see logs for details</div>', 500
 
 
+@pages_v3.route('/settings/search-index')
+def settings_search_index():
+    """Return a flat JSON index of every searchable setting (core + plugin).
+
+    Powers the web UI's global settings search. Built by rendering the settings
+    partials server-side and extracting field metadata, then cached per
+    installed-plugin set so it is off the display's hot path.
+    """
+    # Core settings tabs: (activeTab value, human label, loader).
+    core_tabs = [
+        ('general', 'General', _load_general_partial),
+        ('display', 'Display', _load_display_partial),
+        ('schedule', 'Schedule', _load_schedule_partial),
+        ('wifi', 'WiFi', _load_wifi_partial),
+    ]
+    try:
+        plugin_ids = []
+        if pages_v3.plugin_manager:
+            try:
+                pages_v3.plugin_manager.discover_plugins()
+                plugin_ids = sorted(
+                    pi.get('id') for pi in pages_v3.plugin_manager.get_all_plugin_info()
+                    if pi.get('id')
+                )
+            except Exception:
+                logger.warning("search-index: could not enumerate plugins", exc_info=True)
+
+        sig = tuple(plugin_ids)
+        if _SEARCH_INDEX_CACHE['sig'] == sig and _SEARCH_INDEX_CACHE['fields'] is not None:
+            return jsonify({'fields': _SEARCH_INDEX_CACHE['fields']})
+
+        fields = []
+        for tab, label, loader in core_tabs:
+            fields.extend(_extract_settings_fields(_partial_html(loader), tab, label))
+
+        for pid in plugin_ids:
+            info = pages_v3.plugin_manager.get_plugin_info(pid) or {}
+            label = info.get('name', pid)
+            html = _partial_html(lambda pid=pid: _load_plugin_config_partial(pid))
+            fields.extend(_extract_settings_fields(html, pid, label))
+
+        _SEARCH_INDEX_CACHE['sig'] = sig
+        _SEARCH_INDEX_CACHE['fields'] = fields
+        return jsonify({'fields': fields})
+    except Exception:
+        logger.error("Error building settings search index", exc_info=True)
+        return jsonify({'fields': []}), 500
+
+
 @pages_v3.route('/plugin-ui/<plugin_id>/web-ui/<path:filename>')
 def serve_plugin_web_ui(plugin_id, filename):
     """Serve a plugin's web_ui/ HTML fragment as a standalone page.

web_interface/static/v3/js/settings-search.js

@@ -22,24 +22,8 @@
     if (window._settingsSearchInit) return;
     window._settingsSearchInit = true;
 
-    // Core settings tabs to index. Only tabs that contain real settings fields
-    // (a .form-group[id^="setting-"]) are listed. Operational/action tabs
-    // (overview, logs, history, cache, tools, fonts, backup/restore, raw config
-    // editor, plugin store) have nothing to navigate to and are excluded.
-    var CORE_TABS = [
-        { tab: 'general', label: 'General', url: '/v3/partials/general' },
-        { tab: 'display', label: 'Display', url: '/v3/partials/display' },
-        { tab: 'schedule', label: 'Schedule', url: '/v3/partials/schedule' },
-        { tab: 'wifi', label: 'WiFi', url: '/v3/partials/wifi' }
-    ];
-
     var MAX_RESULTS = 25;
 
-    // Plugin ids are constrained to this allowlist (mirrors the server's
-    // _SAFE_PLUGIN_ID_RE in pages_v3.py) before they can appear in a request
-    // path, so a fetched URL can never be influenced by untrusted input.
-    var PLUGIN_ID_RE = /^[a-zA-Z0-9_-]{1,64}$/;
-
     function debounce(fn, ms) {
         var t;
         return function () {
@@ -60,75 +44,29 @@
 
     // --- Index building -------------------------------------------------------
 
-    // Extract one index entry per settings field from a parsed document.
-    function scanDoc(doc, tab, tabLabel) {
-        var entries = [];
-        var nodes = doc.querySelectorAll('h2, h3, h4, .form-group[id^="setting-"]');
-        var section = '';
-        nodes.forEach(function (node) {
-            var tag = node.tagName.toLowerCase();
-            if (tag === 'h3' || tag === 'h4') {
-                section = textOf(node);
-                return;
-            }
-            if (tag === 'h2') { section = ''; return; }
-            // A settings .form-group
-            var labelEl = node.querySelector('label');
-            var label = textOf(labelEl) || node.id.replace(/^setting-/, '');
-            var tipEl = node.querySelector('.help-tip');
-            var help = tipEl ? (tipEl.getAttribute('data-tooltip') || '') : '';
-            var key = node.getAttribute('data-setting-key') || node.id.replace(/^setting-/, '');
-            entries.push({
-                tab: tab,
-                tabLabel: tabLabel,
-                section: section,
-                key: key,
-                label: label,
-                help: help,
-                anchorId: node.id,
-                hay: [label, help, key, tabLabel, section].join(' ').toLowerCase()
-            });
-        });
-        return entries;
-    }
-
-    function fetchAndScan(url, tab, tabLabel) {
-        // `url` is always one of our own same-origin settings partial paths
-        // (CORE_TABS literals or a plugin path built from an allowlisted id).
-        return fetch(url, { headers: { 'X-Requested-With': 'settings-search' } })
-            .then(function (r) { return r.ok ? r.text() : ''; })
-            .then(function (html) {
-                if (!html) return [];
-                // Parsed into an inert document: scripts do not run and it is
-                // never inserted into the live DOM — we only read labels and
-                // tooltip text from it to build the search index.
-                var doc = new DOMParser().parseFromString(html, 'text/html');
-                return scanDoc(doc, tab, tabLabel);
-            })
-            .catch(function () { return []; });
-    }
-
     var buildPromise = null;
+
+    // Fetch the prebuilt index from the server (one literal-URL JSON request)
+    // and cache it for the session. Each entry gets a lowercased `hay` haystack
+    // for matching. The server owns which tabs/plugins are included.
     function buildIndex(force) {
         if (window._settingsIndex && !force) return Promise.resolve(window._settingsIndex);
         if (buildPromise && !force) return buildPromise;
 
-        var jobs = CORE_TABS.map(function (t) { return fetchAndScan(t.url, t.tab, t.label); });
-
-        var plugins = (window.installedPlugins || []);
-        plugins.forEach(function (p) {
-            // Skip ids that don't match the strict allowlist so the request
-            // path is always built from safe, validated components.
-            if (!p || !p.id || !PLUGIN_ID_RE.test(p.id)) return;
-            jobs.push(fetchAndScan('/v3/partials/plugin-config/' + p.id, p.id, p.name || p.id));
-        });
-
-        buildPromise = Promise.all(jobs).then(function (lists) {
-            var index = [];
-            lists.forEach(function (l) { index = index.concat(l); });
-            window._settingsIndex = index;
-            return index;
-        });
+        buildPromise = fetch('/v3/settings/search-index', { headers: { 'X-Requested-With': 'settings-search' } })
+            .then(function (r) { return r.ok ? r.json() : { fields: [] }; })
+            .then(function (data) {
+                var fields = (data && data.fields) || [];
+                fields.forEach(function (f) {
+                    f.hay = [f.label, f.help, f.key, f.tabLabel, f.section].join(' ').toLowerCase();
+                });
+                window._settingsIndex = fields;
+                return fields;
+            })
+            .catch(function () {
+                window._settingsIndex = [];
+                return window._settingsIndex;
+            });
         return buildPromise;
     }