perf(fetch): cache ESPN scoreboard windows without the parts nothing reads

The sports scoreboards cache their Recent/Upcoming window as the raw ESPN
response, and it stays parsed in the memory tier while fresh. Measured on
hdpi, most of it is never drawn: per-team stat leaders, athlete cards
(featuredAthletes, probables), team and event links, headlines, video
highlights and geo broadcasts. None of those keys is read by core or by any
plugin in ledmatrix-plugins.

BackgroundDataService now drops them from an ESPN /scoreboard response
before caching and delivering it (src/common/espn_payload.py), keeping
everything else. On the five hdpi windows that is 10.6MB -> 3.0MB of JSON
and ~40MB -> ~12MB of parsed objects, and parsing an expired window gets
3-4x cheaper. submit_fetch_request(slim_payload=False) caches a response
whole.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BkfgXMqqwn2w4NN7LRzhxy

src/background_data_service.py

@@ -34,6 +34,7 @@ from src.common.fetch_service import (
     plugin_scope,
     share_connection_pool,
 )
+from src.common.espn_payload import is_espn_scoreboard_url, slim_scoreboard_payload
 from src.common.espn_dates import (
     RANGE_RETRY_SECONDS,
     _note_range_rejected,
@@ -83,6 +84,10 @@ class FetchRequest:
     # the cache with the callbacks suppressed -- joiners waiting forever for a
     # fetch that did, in fact, succeed.
     commit_claimed: bool = False
+    # Trim an ESPN scoreboard response before it is cached and delivered
+    # (src/common/espn_payload.py). Set by whoever created the request; a
+    # submitter that joins the fetch gets the same payload.
+    slim_payload: bool = True
     result: Optional[Any] = None
     error: Optional[str] = None
     # The plugin that submitted the request, so the fetch service counts the
@@ -249,7 +254,8 @@ class BackgroundDataService:
                            timeout: Optional[int] = None,
                            max_retries: int = 3,
                            priority: int = 1,
-                           callback: Optional[Callable] = None) -> str:
+                           callback: Optional[Callable] = None,
+                           slim_payload: bool = True) -> str:
         """
         Submit a background fetch request.
         
@@ -265,6 +271,11 @@ class BackgroundDataService:
             priority: Accepted for compatibility and ignored; requests run in
                 submission order.
             callback: Optional callback function when request completes
+            slim_payload: Drop the parts of an ESPN scoreboard response no
+                scoreboard reads (stat leaders, athlete cards, links,
+                headlines, highlights) before caching it; see
+                src/common/espn_payload.py. Only ESPN /scoreboard URLs are
+                touched. Pass False to cache the response whole.
             
         Returns:
             Request ID for tracking the fetch operation
@@ -336,6 +347,7 @@ class BackgroundDataService:
             priority=priority,
             callback=callback,
             owner=owner,
+            slim_payload=slim_payload,
         )
         
         with self._lock:
@@ -497,6 +509,13 @@ class BackgroundDataService:
                 )
                 return result
 
+            # Most of an ESPN scoreboard response is never drawn, and the
+            # cached copy stays parsed in the memory tier while it is fresh.
+            # Trimmed before the write so the cache, request.result and the
+            # callbacks all see the same payload. See src/common/espn_payload.py.
+            if request.slim_payload and is_espn_scoreboard_url(request.url):
+                slim_scoreboard_payload(data)
+
             # Cache the data
             self.cache_manager.set(request.cache_key, data)
             

src/common/espn_payload.py

@@ -0,0 +1,97 @@
+"""Drop the parts of an ESPN scoreboard payload no scoreboard reads.
+
+The sports scoreboards cache their Recent/Upcoming window (14 days back, 7
+ahead) as the raw ESPN response, and that record stays parsed in the memory
+cache for as long as it is fresh. Most of it is never drawn. Measured on hdpi
+(2026-10-02) the MLB window was 3.35MB of JSON and 13.5MB of Python objects,
+and the five windows together ~40MB, mostly in:
+
+* ``competitors[].leaders`` / ``competitions[].leaders`` -- per-team and
+  per-game stat leaders (28% of the MLB window)
+* ``competitors[].team.links`` / ``event.links`` -- web and app URLs
+* ``status.featuredAthletes`` and ``competitors[].probables`` -- athlete
+  cards with headshots and season stats
+* ``competitions[].headlines`` / ``highlights`` -- article and video blurbs
+  (28% of the college-football window)
+* ``competitions[].geoBroadcasts``
+
+None of those keys is read by core or by any plugin in ledmatrix-plugins
+(checked 2026-10-02 across every scoreboard, the odds ticker and the
+leaderboard), while everything that is read -- odds, records, linescores,
+situation, statistics, notes, broadcasts, venue -- is kept. Dropping them
+takes the five windows from ~40MB to ~12MB of parsed objects and the files from
+10.6MB to 3.0MB, so the reads that parse an expired window on the render
+thread get 3-4x cheaper too.
+
+:func:`slim_scoreboard_payload` changes the payload in place, and only ever
+removes the keys listed here: anything it does not know about is left alone.
+"""
+
+from typing import Any, Dict
+from urllib.parse import urlsplit
+
+# Per level of the payload, the keys removed. Kept deliberately explicit:
+# adding a key here means checking that nothing reads it first.
+_EVENT_DROP = ("links",)
+_COMPETITION_DROP = ("leaders", "headlines", "highlights", "geoBroadcasts")
+_STATUS_DROP = ("featuredAthletes",)
+_COMPETITOR_DROP = ("leaders", "probables")
+_TEAM_DROP = ("links",)
+
+
+def is_espn_scoreboard_url(url: Any) -> bool:
+    """Whether ``url`` is an ESPN site-API scoreboard endpoint."""
+    if not isinstance(url, str):
+        return False
+    try:
+        parts = urlsplit(url)
+    except ValueError:
+        return False
+    host = (parts.hostname or "").lower()
+    if host != "espn.com" and not host.endswith(".espn.com"):
+        return False
+    return parts.path.rstrip("/").endswith("/scoreboard")
+
+
+def _drop(obj: Any, keys) -> None:
+    if isinstance(obj, dict):
+        for key in keys:
+            obj.pop(key, None)
+
+
+def slim_scoreboard_payload(payload: Any) -> Any:
+    """Remove the unread parts of an ESPN scoreboard payload, in place.
+
+    Returns ``payload`` for convenience. Anything that is not shaped like a
+    scoreboard (not a dict, no ``events`` list, odd entries) is passed over
+    untouched rather than raising.
+    """
+    if not isinstance(payload, dict):
+        return payload
+    events = payload.get("events")
+    if not isinstance(events, list):
+        return payload
+    for event in events:
+        if not isinstance(event, dict):
+            continue
+        _drop(event, _EVENT_DROP)
+        competitions = event.get("competitions")
+        if not isinstance(competitions, list):
+            continue
+        for competition in competitions:
+            if not isinstance(competition, dict):
+                continue
+            _drop(competition, _COMPETITION_DROP)
+            _drop(competition.get("status"), _STATUS_DROP)
+            competitors = competition.get("competitors")
+            if not isinstance(competitors, list):
+                continue
+            for competitor in competitors:
+                if not isinstance(competitor, dict):
+                    continue
+                _drop(competitor, _COMPETITOR_DROP)
+                _drop(competitor.get("team"), _TEAM_DROP)
+    return payload
+
+
+__all__ = ["is_espn_scoreboard_url", "slim_scoreboard_payload"]

test/test_espn_payload.py

@@ -0,0 +1,162 @@
+"""Tests for src/common/espn_payload.py and its use by BackgroundDataService."""
+
+import copy
+import time
+from unittest.mock import MagicMock, Mock, patch
+
+import pytest
+
+from src.background_data_service import BackgroundDataService, shutdown_background_service
+from src.common.espn_payload import is_espn_scoreboard_url, slim_scoreboard_payload
+
+SCOREBOARD = "https://site.api.espn.com/apis/site/v2/sports/baseball/mlb/scoreboard"
+
+
+def _event():
+    """One event carrying every key the slimming drops and a sample of the
+    keys scoreboards read, at the depth ESPN puts them."""
+    competitor = {
+        "id": "10",
+        "homeAway": "home",
+        "score": "5",
+        "team": {"abbreviation": "NYY", "logo": "https://a/l.png",
+                 "links": [{"href": "https://espn.com/team"}]},
+        "records": [{"summary": "90-60"}],
+        "linescores": [{"value": 1}],
+        "statistics": [{"name": "hits", "displayValue": "9"}],
+        "leaders": [{"name": "avg", "leaders": [{"athlete": {"id": "1"}}]}],
+        "probables": [{"athlete": {"id": "2"}, "statistics": []}],
+    }
+    return {
+        "id": "401",
+        "date": "2026-10-01T23:05Z",
+        "links": [{"href": "https://espn.com/game"}],
+        "status": {"type": {"state": "post"}},
+        "competitions": [{
+            "status": {"type": {"state": "post", "shortDetail": "Final"},
+                       "featuredAthletes": [{"athlete": {"id": "3"}}]},
+            "competitors": [competitor, dict(copy.deepcopy(competitor), homeAway="away")],
+            "odds": [{"details": "NYY -150", "overUnder": 8.5}],
+            "situation": {"outs": 2},
+            "notes": [{"headline": "Game 1"}],
+            "broadcasts": [{"names": ["FOX"]}],
+            "venue": {"fullName": "Yankee Stadium"},
+            "leaders": [{"name": "hits"}],
+            "headlines": [{"description": "recap"}],
+            "highlights": [{"links": {"source": {}}}],
+            "geoBroadcasts": [{"media": {"shortName": "FOX"}}],
+        }],
+    }
+
+
+class TestSlimScoreboardPayload:
+    def test_drops_exactly_the_listed_keys(self):
+        payload = {"leagues": [{"id": "10"}], "events": [_event()]}
+        slim_scoreboard_payload(payload)
+        event = payload["events"][0]
+        competition = event["competitions"][0]
+        assert "links" not in event
+        for key in ("leaders", "headlines", "highlights", "geoBroadcasts"):
+            assert key not in competition
+        assert "featuredAthletes" not in competition["status"]
+        for competitor in competition["competitors"]:
+            assert "leaders" not in competitor
+            assert "probables" not in competitor
+            assert "links" not in competitor["team"]
+
+    def test_keeps_everything_else_unchanged(self):
+        """Removing the dropped keys from the original by hand gives exactly
+        the slimmed payload: nothing else moved, changed or went missing."""
+        original = {"leagues": [{"id": "10"}], "events": [_event(), _event()]}
+        expected = copy.deepcopy(original)
+        for event in expected["events"]:
+            del event["links"]
+            competition = event["competitions"][0]
+            for key in ("leaders", "headlines", "highlights", "geoBroadcasts"):
+                del competition[key]
+            del competition["status"]["featuredAthletes"]
+            for competitor in competition["competitors"]:
+                del competitor["leaders"], competitor["probables"]
+                del competitor["team"]["links"]
+        assert slim_scoreboard_payload(original) == expected
+
+    def test_in_place_and_returns_payload(self):
+        payload = {"events": [_event()]}
+        assert slim_scoreboard_payload(payload) is payload
+
+    @pytest.mark.parametrize("payload", [
+        None, [], "x", {}, {"events": None}, {"events": "x"},
+        {"events": [None, 1, "x", {"competitions": None}]},
+        {"events": [{"competitions": [None, {"status": None, "competitors": None}]}]},
+        {"events": [{"competitions": [{"competitors": [None, {"team": None}]}]}]},
+    ])
+    def test_odd_shapes_pass_through(self, payload):
+        before = copy.deepcopy(payload)
+        assert slim_scoreboard_payload(payload) == before
+
+
+class TestIsEspnScoreboardUrl:
+    @pytest.mark.parametrize("url", [
+        SCOREBOARD,
+        SCOREBOARD + "/",
+        "http://site.api.espn.com/apis/site/v2/sports/football/college-football/scoreboard",
+    ])
+    def test_scoreboards(self, url):
+        assert is_espn_scoreboard_url(url)
+
+    @pytest.mark.parametrize("url", [
+        None, "", 12,
+        "https://site.api.espn.com/apis/site/v2/sports/baseball/mlb/teams",
+        "https://site.api.espn.com/apis/site/v2/sports/football/nfl/summary",
+        "https://example.com/scoreboard",
+        "https://espn.com.evil.example/apis/x/scoreboard",
+        "https://notespn.com/apis/x/scoreboard",
+    ])
+    def test_not_scoreboards(self, url):
+        assert not is_espn_scoreboard_url(url)
+
+
+@pytest.fixture
+def service():
+    shutdown_background_service()
+    cache = MagicMock()
+    cache.get.return_value = None
+    svc = BackgroundDataService(cache, max_workers=1, request_timeout=5)
+    yield svc
+    svc.shutdown(wait=False)
+    shutdown_background_service()
+
+
+def _run(service, url, **kwargs):
+    response = Mock(status_code=200)
+    response.json.return_value = {"events": [_event()]}
+    response.raise_for_status.return_value = None
+    delivered = []
+    with patch.object(service.session, "get", return_value=response):
+        req_id = service.submit_fetch_request(
+            sport="mlb", year=2026, url=url, cache_key="mlb_schedule_window_14_7",
+            callback=lambda result: delivered.append(result.data), **kwargs)
+        deadline = time.time() + 5
+        while not service.is_request_complete(req_id) and time.time() < deadline:
+            time.sleep(0.02)
+    cached = service.cache_manager.set.call_args[0][1]
+    return cached, delivered
+
+
+class TestBackgroundServiceSlims:
+    def test_espn_scoreboard_is_cached_and_delivered_slimmed(self, service):
+        cached, delivered = _run(service, SCOREBOARD)
+        competition = cached["events"][0]["competitions"][0]
+        assert "leaders" not in competition
+        assert "probables" not in competition["competitors"][0]
+        assert competition["odds"] and competition["situation"]
+        # The callback sees the very payload that was cached.
+        assert delivered and delivered[0] is cached
+
+    def test_opt_out_caches_whole_response(self, service):
+        cached, _ = _run(service, SCOREBOARD, slim_payload=False)
+        assert cached == {"events": [_event()]}
+
+    def test_other_urls_untouched(self, service):
+        cached, _ = _run(service, "https://example.com/feed")
+        assert cached == {"events": [_event()]}