diff --git a/src/background_data_service.py b/src/background_data_service.py index be860bb9..9a8f0250 100644 --- a/src/background_data_service.py +++ b/src/background_data_service.py @@ -34,6 +34,7 @@ from src.common.fetch_service import ( plugin_scope, share_connection_pool, ) +from src.common.espn_payload import is_espn_scoreboard_url, slim_scoreboard_payload from src.common.espn_dates import ( RANGE_RETRY_SECONDS, _note_range_rejected, @@ -83,6 +84,10 @@ class FetchRequest: # the cache with the callbacks suppressed -- joiners waiting forever for a # fetch that did, in fact, succeed. commit_claimed: bool = False + # Trim an ESPN scoreboard response before it is cached and delivered + # (src/common/espn_payload.py). Set by whoever created the request; a + # submitter that joins the fetch gets the same payload. + slim_payload: bool = True result: Optional[Any] = None error: Optional[str] = None # The plugin that submitted the request, so the fetch service counts the @@ -249,7 +254,8 @@ class BackgroundDataService: timeout: Optional[int] = None, max_retries: int = 3, priority: int = 1, - callback: Optional[Callable] = None) -> str: + callback: Optional[Callable] = None, + slim_payload: bool = True) -> str: """ Submit a background fetch request. @@ -265,6 +271,11 @@ class BackgroundDataService: priority: Accepted for compatibility and ignored; requests run in submission order. callback: Optional callback function when request completes + slim_payload: Drop the parts of an ESPN scoreboard response no + scoreboard reads (stat leaders, athlete cards, links, + headlines, highlights) before caching it; see + src/common/espn_payload.py. Only ESPN /scoreboard URLs are + touched. Pass False to cache the response whole. Returns: Request ID for tracking the fetch operation @@ -336,6 +347,7 @@ class BackgroundDataService: priority=priority, callback=callback, owner=owner, + slim_payload=slim_payload, ) with self._lock: @@ -497,6 +509,13 @@ class BackgroundDataService: ) return result + # Most of an ESPN scoreboard response is never drawn, and the + # cached copy stays parsed in the memory tier while it is fresh. + # Trimmed before the write so the cache, request.result and the + # callbacks all see the same payload. See src/common/espn_payload.py. + if request.slim_payload and is_espn_scoreboard_url(request.url): + slim_scoreboard_payload(data) + # Cache the data self.cache_manager.set(request.cache_key, data) diff --git a/src/common/espn_payload.py b/src/common/espn_payload.py new file mode 100644 index 00000000..00ff118c --- /dev/null +++ b/src/common/espn_payload.py @@ -0,0 +1,97 @@ +"""Drop the parts of an ESPN scoreboard payload no scoreboard reads. + +The sports scoreboards cache their Recent/Upcoming window (14 days back, 7 +ahead) as the raw ESPN response, and that record stays parsed in the memory +cache for as long as it is fresh. Most of it is never drawn. Measured on hdpi +(2026-10-02) the MLB window was 3.35MB of JSON and 13.5MB of Python objects, +and the five windows together ~40MB, mostly in: + +* ``competitors[].leaders`` / ``competitions[].leaders`` -- per-team and + per-game stat leaders (28% of the MLB window) +* ``competitors[].team.links`` / ``event.links`` -- web and app URLs +* ``status.featuredAthletes`` and ``competitors[].probables`` -- athlete + cards with headshots and season stats +* ``competitions[].headlines`` / ``highlights`` -- article and video blurbs + (28% of the college-football window) +* ``competitions[].geoBroadcasts`` + +None of those keys is read by core or by any plugin in ledmatrix-plugins +(checked 2026-10-02 across every scoreboard, the odds ticker and the +leaderboard), while everything that is read -- odds, records, linescores, +situation, statistics, notes, broadcasts, venue -- is kept. Dropping them +takes the five windows from ~40MB to ~12MB of parsed objects and the files from +10.6MB to 3.0MB, so the reads that parse an expired window on the render +thread get 3-4x cheaper too. + +:func:`slim_scoreboard_payload` changes the payload in place, and only ever +removes the keys listed here: anything it does not know about is left alone. +""" + +from typing import Any, Dict +from urllib.parse import urlsplit + +# Per level of the payload, the keys removed. Kept deliberately explicit: +# adding a key here means checking that nothing reads it first. +_EVENT_DROP = ("links",) +_COMPETITION_DROP = ("leaders", "headlines", "highlights", "geoBroadcasts") +_STATUS_DROP = ("featuredAthletes",) +_COMPETITOR_DROP = ("leaders", "probables") +_TEAM_DROP = ("links",) + + +def is_espn_scoreboard_url(url: Any) -> bool: + """Whether ``url`` is an ESPN site-API scoreboard endpoint.""" + if not isinstance(url, str): + return False + try: + parts = urlsplit(url) + except ValueError: + return False + host = (parts.hostname or "").lower() + if host != "espn.com" and not host.endswith(".espn.com"): + return False + return parts.path.rstrip("/").endswith("/scoreboard") + + +def _drop(obj: Any, keys) -> None: + if isinstance(obj, dict): + for key in keys: + obj.pop(key, None) + + +def slim_scoreboard_payload(payload: Any) -> Any: + """Remove the unread parts of an ESPN scoreboard payload, in place. + + Returns ``payload`` for convenience. Anything that is not shaped like a + scoreboard (not a dict, no ``events`` list, odd entries) is passed over + untouched rather than raising. + """ + if not isinstance(payload, dict): + return payload + events = payload.get("events") + if not isinstance(events, list): + return payload + for event in events: + if not isinstance(event, dict): + continue + _drop(event, _EVENT_DROP) + competitions = event.get("competitions") + if not isinstance(competitions, list): + continue + for competition in competitions: + if not isinstance(competition, dict): + continue + _drop(competition, _COMPETITION_DROP) + _drop(competition.get("status"), _STATUS_DROP) + competitors = competition.get("competitors") + if not isinstance(competitors, list): + continue + for competitor in competitors: + if not isinstance(competitor, dict): + continue + _drop(competitor, _COMPETITOR_DROP) + _drop(competitor.get("team"), _TEAM_DROP) + return payload + + +__all__ = ["is_espn_scoreboard_url", "slim_scoreboard_payload"] diff --git a/test/test_espn_payload.py b/test/test_espn_payload.py new file mode 100644 index 00000000..1605c586 --- /dev/null +++ b/test/test_espn_payload.py @@ -0,0 +1,162 @@ +"""Tests for src/common/espn_payload.py and its use by BackgroundDataService.""" + +import copy +import time +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from src.background_data_service import BackgroundDataService, shutdown_background_service +from src.common.espn_payload import is_espn_scoreboard_url, slim_scoreboard_payload + +SCOREBOARD = "https://site.api.espn.com/apis/site/v2/sports/baseball/mlb/scoreboard" + + +def _event(): + """One event carrying every key the slimming drops and a sample of the + keys scoreboards read, at the depth ESPN puts them.""" + competitor = { + "id": "10", + "homeAway": "home", + "score": "5", + "team": {"abbreviation": "NYY", "logo": "https://a/l.png", + "links": [{"href": "https://espn.com/team"}]}, + "records": [{"summary": "90-60"}], + "linescores": [{"value": 1}], + "statistics": [{"name": "hits", "displayValue": "9"}], + "leaders": [{"name": "avg", "leaders": [{"athlete": {"id": "1"}}]}], + "probables": [{"athlete": {"id": "2"}, "statistics": []}], + } + return { + "id": "401", + "date": "2026-10-01T23:05Z", + "links": [{"href": "https://espn.com/game"}], + "status": {"type": {"state": "post"}}, + "competitions": [{ + "status": {"type": {"state": "post", "shortDetail": "Final"}, + "featuredAthletes": [{"athlete": {"id": "3"}}]}, + "competitors": [competitor, dict(copy.deepcopy(competitor), homeAway="away")], + "odds": [{"details": "NYY -150", "overUnder": 8.5}], + "situation": {"outs": 2}, + "notes": [{"headline": "Game 1"}], + "broadcasts": [{"names": ["FOX"]}], + "venue": {"fullName": "Yankee Stadium"}, + "leaders": [{"name": "hits"}], + "headlines": [{"description": "recap"}], + "highlights": [{"links": {"source": {}}}], + "geoBroadcasts": [{"media": {"shortName": "FOX"}}], + }], + } + + +class TestSlimScoreboardPayload: + def test_drops_exactly_the_listed_keys(self): + payload = {"leagues": [{"id": "10"}], "events": [_event()]} + slim_scoreboard_payload(payload) + event = payload["events"][0] + competition = event["competitions"][0] + assert "links" not in event + for key in ("leaders", "headlines", "highlights", "geoBroadcasts"): + assert key not in competition + assert "featuredAthletes" not in competition["status"] + for competitor in competition["competitors"]: + assert "leaders" not in competitor + assert "probables" not in competitor + assert "links" not in competitor["team"] + + def test_keeps_everything_else_unchanged(self): + """Removing the dropped keys from the original by hand gives exactly + the slimmed payload: nothing else moved, changed or went missing.""" + original = {"leagues": [{"id": "10"}], "events": [_event(), _event()]} + expected = copy.deepcopy(original) + for event in expected["events"]: + del event["links"] + competition = event["competitions"][0] + for key in ("leaders", "headlines", "highlights", "geoBroadcasts"): + del competition[key] + del competition["status"]["featuredAthletes"] + for competitor in competition["competitors"]: + del competitor["leaders"], competitor["probables"] + del competitor["team"]["links"] + assert slim_scoreboard_payload(original) == expected + + def test_in_place_and_returns_payload(self): + payload = {"events": [_event()]} + assert slim_scoreboard_payload(payload) is payload + + @pytest.mark.parametrize("payload", [ + None, [], "x", {}, {"events": None}, {"events": "x"}, + {"events": [None, 1, "x", {"competitions": None}]}, + {"events": [{"competitions": [None, {"status": None, "competitors": None}]}]}, + {"events": [{"competitions": [{"competitors": [None, {"team": None}]}]}]}, + ]) + def test_odd_shapes_pass_through(self, payload): + before = copy.deepcopy(payload) + assert slim_scoreboard_payload(payload) == before + + +class TestIsEspnScoreboardUrl: + @pytest.mark.parametrize("url", [ + SCOREBOARD, + SCOREBOARD + "/", + "http://site.api.espn.com/apis/site/v2/sports/football/college-football/scoreboard", + ]) + def test_scoreboards(self, url): + assert is_espn_scoreboard_url(url) + + @pytest.mark.parametrize("url", [ + None, "", 12, + "https://site.api.espn.com/apis/site/v2/sports/baseball/mlb/teams", + "https://site.api.espn.com/apis/site/v2/sports/football/nfl/summary", + "https://example.com/scoreboard", + "https://espn.com.evil.example/apis/x/scoreboard", + "https://notespn.com/apis/x/scoreboard", + ]) + def test_not_scoreboards(self, url): + assert not is_espn_scoreboard_url(url) + + +@pytest.fixture +def service(): + shutdown_background_service() + cache = MagicMock() + cache.get.return_value = None + svc = BackgroundDataService(cache, max_workers=1, request_timeout=5) + yield svc + svc.shutdown(wait=False) + shutdown_background_service() + + +def _run(service, url, **kwargs): + response = Mock(status_code=200) + response.json.return_value = {"events": [_event()]} + response.raise_for_status.return_value = None + delivered = [] + with patch.object(service.session, "get", return_value=response): + req_id = service.submit_fetch_request( + sport="mlb", year=2026, url=url, cache_key="mlb_schedule_window_14_7", + callback=lambda result: delivered.append(result.data), **kwargs) + deadline = time.time() + 5 + while not service.is_request_complete(req_id) and time.time() < deadline: + time.sleep(0.02) + cached = service.cache_manager.set.call_args[0][1] + return cached, delivered + + +class TestBackgroundServiceSlims: + def test_espn_scoreboard_is_cached_and_delivered_slimmed(self, service): + cached, delivered = _run(service, SCOREBOARD) + competition = cached["events"][0]["competitions"][0] + assert "leaders" not in competition + assert "probables" not in competition["competitors"][0] + assert competition["odds"] and competition["situation"] + # The callback sees the very payload that was cached. + assert delivered and delivered[0] is cached + + def test_opt_out_caches_whole_response(self, service): + cached, _ = _run(service, SCOREBOARD, slim_payload=False) + assert cached == {"events": [_event()]} + + def test_other_urls_untouched(self, service): + cached, _ = _run(service, "https://example.com/feed") + assert cached == {"events": [_event()]}