From f9b0b237a3120e5821cbf65ceb8b19229c79a652 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:21:50 -0400 Subject: [PATCH] feat(common): sports_live_fetch -- the reconciled live fetch and odds (sports family 9) (#793) New hardware-free module src/common/sports_live_fetch.py, copied from ledmatrix-plugins claude/family9-reconcile once the nine scoreboards made _fetch_todays_games (seven bodies when measured) and _fetch_odds (three) one body each. Both live on the plugins' SportsCore, so one mixin, SportsLiveFetchMixin, carries exactly those two plus the odds worker _drain_odds_queue; no manager gains a method it lacks today. Owner decisions (2026-10-12): - Live scores are always fresh: every poll asks ESPN through get_espn_scoreboard and leaves the answer under the shared espn_scoreboard_cache_key for its dates (max_age=1: only a copy another reader fetched within the last second is reused). The plugins' 30 s _scoreboard_current cache is gone. - College basketball is asked by date like every other league (ESPN answered single dates for both college leagues on 2026-10-12). - Odds never block update(): one daemon worker per manager answers a queue and attaches each line in place under _games_lock; one batch in flight per manager, games queued meanwhile join it, every dict queued for an id gets the one answer. show_odds, the two odds intervals and a missing odds manager (debug, not ERROR) are respected. - An answer whose events is not a list is no games. Fix: get_espn_scoreboard() no longer caches an answer whose events is not a list ("events": null), which every reader of the shared key inherited; it is still returned to the caller. - test/test_sports_live_fetch.py: the plugins' pinned cases (requests, dates and keys at 10:00 and 02:00 Eastern, freshness, malformed and failed answers, odds that never block and land later, the intervals, one batch per manager, a worker that cannot start, update and display on two threads), plus the host contract. Mutation-checked (11 mutants, all caught). - test/test_sports_live_fetch_parity.py: with LEDMATRIX_PLUGINS, compares each body with every plugin copy. Passes against claude/family9-reconcile; fails against the pre-reconcile tree. - test/test_espn_scoreboard_cache.py: the null-events regression. - mypy ratchet, src/common/README.md, CHANGELOG (Unreleased: New modules, Fixes), docs/SPORTS_UNIFICATION.md family 9 status and decisions. Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 20 ++ docs/SPORTS_UNIFICATION.md | 70 ++++- mypy-clean.txt | 1 + src/common/README.md | 14 + src/common/espn_dates.py | 7 +- src/common/sports_live_fetch.py | 200 +++++++++++++ test/test_espn_scoreboard_cache.py | 9 + test/test_sports_live_fetch.py | 399 ++++++++++++++++++++++++++ test/test_sports_live_fetch_parity.py | 104 +++++++ 9 files changed, 812 insertions(+), 12 deletions(-) create mode 100644 src/common/sports_live_fetch.py create mode 100644 test/test_sports_live_fetch.py create mode 100644 test/test_sports_live_fetch_parity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e4f752e..fd80c432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,28 @@ accepts both, but the store flags the old spelling as deprecated web interface still starts on boot either way. Hidden when the unit's state cannot be read (not installed, or no systemd). +### New modules + +- `src/common/sports_live_fetch.py` -- sports consolidation family 9, once the + plugins made `_fetch_todays_games` (seven bodies) and `_fetch_odds` (three) + one each: `SportsLiveFetchMixin` (`SportsCore`), with the odds worker + `_drain_odds_queue` beside them. The live poll asks ESPN on every poll + (a copy another reader fetched within the last second is reused) and + leaves the answer under the shared `espn_scoreboard_cache_key` for its + dates; college basketball is asked by date like every other league; an + answer whose `events` is not a list counts as no games. Odds never block + `update()`: one daemon worker per manager answers a queue and attaches each + line in place under `_games_lock`, one batch in flight at a time. A plugin + may inherit the mixin once it floors on the release that ships it, and + deletes its copies then. + ### Fixes +- `get_espn_scoreboard()` no longer caches an answer whose `events` is not a + list (ESPN has sent `"events": null`): every reader of the shared scoreboard + key -- odds-ticker, the other scoreboards -- inherited it until it aged out. + The answer is still returned to the caller as it came. + - `SportsRankingsMixin` (#789's deferred review points, now that it is the only copy): a poll entry whose `current` cannot be converted to an integer (`None`, a non-numeric string) is skipped instead of throwing diff --git a/docs/SPORTS_UNIFICATION.md b/docs/SPORTS_UNIFICATION.md index d89d911b..21c2f735 100644 --- a/docs/SPORTS_UNIFICATION.md +++ b/docs/SPORTS_UNIFICATION.md @@ -415,6 +415,45 @@ its id table too); the two differ only for a poll whose blocks carry ids and no abbreviations. The data sources' `fetch_standings` (held with `data_sources.py`) is still four bodies; only its endpoint order changed. +### Family 9: live fetch and odds (reconciled; promoted, awaiting release) + +ledmatrix-plugins `scripts/test_live_fetch_odds.py` (#649) pinned 44 rows +across the nine plugins first, every ESPN request answered offline the way +ESPN answered on 2026-10-12 (a single date 200, a range 400): the requests, +cache keys and TTLs of `_fetch_todays_games` per league and Eastern time, +the live cache over time, empty, null and failed answers, `_fetch_odds` per +game kind and odds endpoint on a virtual clock (which thread asks, how long +the call waits, when the line lands), the real `update()` of every screen +with a 3 s odds endpoint, and the league overrides (MiLB, HockeyTech, ufc's +`MMALive`, which stay per-sport). Measuring found seven bodies of +`_fetch_todays_games`, not five: the fetch service's shared key had split +the old groups. The reconcile (ledmatrix-plugins #650) +made `_fetch_todays_games` and `_fetch_odds` one body each on `SportsCore`, +with the odds worker `_drain_odds_queue` beside them, and added a test that +runs `update()` and `display()` on two threads against a slow endpoint. +Every changed cell is one of the decisions below; the harness renders were +byte-identical once the four fixtures that seeded the old live key seed the +shared one. `src/common/sports_live_fetch.py` holds the bodies in one mixin, +`SportsLiveFetchMixin`; `test/test_sports_live_fetch_parity.py` compares them +with the plugin copies. + +Measured `update()` time with a 3 s odds endpoint and four games, before and +after: Upcoming 6 s (afl, baseball, football, nrl, soccer, ufc), 12 s +(hockey, lacrosse), 0 s (basketball) to 0 s everywhere; Recent 6 s / 12 s to +0 s; Live's first pass 8 s / 12 s and next poll 4 s / 6 s to 0 s. The line +now lands when ESPN answers (3 s here) instead of at the next `update()` +(15 s for a live game, an hour for an upcoming or recent one) in the six. + +Left for later: ufc's MMA Recent manager never asks for odds and its Upcoming +manager asks for every fight in the window before trimming to the headliners +(both in `mma.py`'s `update()` overrides, family 13); basketball's NCAA +Recent/Upcoming fallback still caches the undated scoreboard under +`_scoreboard_current` with `limit=1000`, which ESPN truncates (held +with the data fetching); `_attach_odds_to_rotated_games` still starts its own +thread per rotation and can route through `_fetch_odds` once every plugin +inherits this mixin; and ESPN lists only some college basketball games +without `groups=50` (7 of 21 on 2026-03-14), dated or not. + ### Why the method changes Byte-identical promotion has nearly run dry. Measured on ledmatrix-plugins @@ -503,7 +542,7 @@ release. | 6 | Favourite matching | `_is_favorite_game` (7 across three classes), `_select_games_for_display` (2: nrl), `_select_recent_games_for_display` (3) | Everything that asks "is this a favourite" goes through the 3.5.0 `_favorite_key` seam. Reconciled to one body each and promoted as `sports_favorites`; shipped in 3.8.2 and adopted. See [Family 6](#family-6-favourite-matching-done-core-382-adopted) | | 7 | Other-games rotation | `_by_importance`, `_other_games_window`, `_advance_other_games_if_due` (2 each: football), `_rotate_other_games_on_display` (2: ufc), with `_attach_odds_to_rotated_games` (3; ufc had none) | One outlier each; football carried two fixes the other eight lacked. Reconciled to one body each, on a `_rankings_loaded` seam, and promoted as `sports_rotation`; done: core 3.8.4, adopted (ledmatrix-plugins #643). See [Family 7](#family-7-the-other-games-rotation-done-core-384-adopted) | | 8 | Rankings | `_fetch_team_rankings` (3), `_choose_poll` (3), `_load_division_team_ids`, `_passes_other_filters`, `_best_rank`, `_is_ranked_game` (2 each: football) | Needs 7; the rank badge and the "ranked only" filter read it. Reconciled to one body each and promoted as `sports_rankings`; done: core 3.8.5, adopted (ledmatrix-plugins #648). See [Family 8](#family-8-rankings-done-core-385-adopted) | -| 9 | Live fetch and odds | `_fetch_todays_games` (5), `_fetch_odds` (3), `_attach_odds_to_rotated_games` (3) | The prerequisite for one shared ESPN poller across plugins | +| 9 | Live fetch and odds | `_fetch_todays_games` (7 when measured), `_fetch_odds` (3); `_attach_odds_to_rotated_games` went with 7 | The prerequisite for one shared ESPN poller across plugins. Reconciled to one body each and promoted as `sports_live_fetch` (next release). See [Family 9](#family-9-live-fetch-and-odds-reconciled-promoted-awaiting-release) | | 10 | View model | `_extract_game_details_common` (9 of 9) | Every renderer reads it; its keys are additive-only, so reconcile to the superset and leave sport extras in `_extract_game_details` | | 11 | Switch scorebug | `_load_fonts` (4), `_load_custom_font_from_element_config` (6), `_get_layout_offset` (5), `_fit_score_font` (2: football), `_load_and_resize_logo` (9), `_draw_dynamic_odds` (9), then `_draw_scorebug_layout` (20: Upcoming 9, Recent 8, Live 2, ufc's Core 1) | Needs 10 and the twin decisions below. Several releases: fonts and offsets, logos, odds, then one mode's layout per release | | 12 | Scroll/Vegas card | `game_renderer.py`: `render_game_card` (6), `_load_and_resize_logo` (8), `_load_custom_font`, `preload_logos`, `__init__` (7 each), `_draw_records_or_rankings` (6), `_load_fonts`, `_draw_dynamic_odds`, `_draw_live_game_status`, `_draw_recent_game_status`, `_get_team_display_text` (5 each), `_draw_text_with_outline` (2: football) | Same decisions as 11; sequence after the scroll-performance work on pre-rendered strips lands | @@ -598,15 +637,26 @@ suspected behaviour that needs a payload or a rig to confirm first. seasons. A block without a readable `endDate` counts as current. (f) baseball's MLB, MiLB and NCAA Baseball managers drop their class-level shared copy of the table (none of those leagues publishes a poll). -- **9, live fetch and odds.** (a) afl, basketball, nrl and soccer cache the - live scoreboard for 30 s under `_scoreboard_current`; the others - do not (the harness fixtures seed that key, so the change shows up there). - (b) basketball fetches college games with no `dates` parameter, citing a - 404 (*verify* now that `espn_dates` handles ranges). (c) odds are fetched - three ways: blocking (hockey, lacrosse), a thread waited on for 1.5–2 s - (six plugins), or fire-and-forget for upcoming games (basketball). This - sets how long `update()` takes and when an odds line appears. (d) nrl - guards on a missing odds manager; port it. +- **9, live fetch and odds. Decided 2026-10-12, reconciled.** (a) Live + scores are always fresh: every poll asks ESPN and leaves the answer under + the shared `espn_scoreboard_cache_key` for its dates (`max_age=1`: only a + copy another reader fetched within the last second, the managers of one + display tick, is reused). afl, basketball, nrl and soccer had held it 30 s + under `_scoreboard_current`, so with the 15 s default poll every + other poll repeated the last one; lacrosse and ufc now share their copy. + (b) College basketball is asked by date like every other league: ESPN + answered single dates for both college leagues on 2026-10-12 (the 404 no + longer happens), so the lookback now reaches last night's late games there + too. `groups=50` is a separate question (left above). (c) Odds never block + `update()`: one daemon worker per manager answers a queue and attaches each + line in place under `_games_lock`; one batch in flight per manager, games + queued meanwhile join it. A card can draw once without odds and gain them; + a pre-rendered scroll strip gains them when it is next rebuilt. (d) The + missing-odds-manager guard (nrl's, already in `_attach_odds_to_rotated_games` + since family 7) is in `_fetch_odds` too, logged at debug, not ERROR. Fix + ported: an answer with `"events": null` is no games instead of a + `TypeError` out of the fetch (all nine), and `get_espn_scoreboard` no longer + caches it under the shared key. - **10, view model.** Per key, whether every sport emits it. Additive only: no key is renamed or removed. - **11 and 12, the scorebug and the card.** The pinned divergences in diff --git a/mypy-clean.txt b/mypy-clean.txt index c88ea3a5..a3f8920f 100644 --- a/mypy-clean.txt +++ b/mypy-clean.txt @@ -39,6 +39,7 @@ src/common/sports_favorites.py src/common/sports_fetch.py src/common/sports_font_path.py src/common/sports_game_over.py +src/common/sports_live_fetch.py src/common/sports_live_scroll.py src/common/sports_plugin_host.py src/common/sports_rankings.py diff --git a/src/common/README.md b/src/common/README.md index 6e8e5882..4168d141 100644 --- a/src/common/README.md +++ b/src/common/README.md @@ -51,6 +51,7 @@ Rules for the package: | [`sports_game_over`](#sports_game_over) | Whether a game ESPN still lists as live has ended | Yes (scoreboards) | 3.8.1 | | [`sports_game_renderer`](#sports_game_renderer) | Scoreboard scroll/Vegas card geometry | Yes (scoreboards) | 3.3.0 | | [`sports_helpers`](#sports_helpers) | Small helpers every scoreboard `sports.py` copies | Yes (scoreboards) | 3.5.0 | +| [`sports_live_fetch`](#sports_live_fetch) | The live poll's scoreboard request, and odds off the update path | Yes (scoreboards) | next release | | [`sports_live_scroll`](#sports_live_scroll) | Rebuild a live scroll strip mid-cycle without moving it | Yes (scoreboards) | 3.8.0 | | [`sports_plugin_host`](#sports_plugin_host) | Helpers of a scoreboard's plugin class (`manager.py`) | Yes (scoreboards) | 3.8.0 | | [`sports_rankings`](#sports_rankings) | Which teams are ranked, in which poll, and the "ranked" filter | Yes (scoreboards) | 3.8.5 | @@ -370,6 +371,19 @@ shape), `_dispatch_switch_refresh()` (a manager refresh on a daemon thread, so `display()` never waits on the network), `get_vegas_content_type()` and small dynamic-duration helpers. List it before `BasePlugin`. +### sports_live_fetch + +[`sports_live_fetch.py`](sports_live_fetch.py). Sports family 9: +`SportsLiveFetchMixin` (`SportsCore`). `_fetch_todays_games` asks ESPN for +today's scoreboard (and yesterday's while `_needs_previous_day()` says it can +still hold a live game) on every live poll through `get_espn_scoreboard`, and +leaves it under the shared `espn_scoreboard_cache_key` for odds-ticker and the +other scoreboards; an answer without an `events` list is no games. +`_fetch_odds` queues a game and returns at once; one daemon worker per manager +(`_drain_odds_queue`) asks the odds manager for each queued game and attaches +the line in place under `_games_lock`, so a card can draw once without odds and +gain them. Games queued while the worker is busy join its batch. + ### sports_rankings [`sports_rankings.py`](sports_rankings.py). Sports family 8: diff --git a/src/common/espn_dates.py b/src/common/espn_dates.py index 5c8080a6..fbb81e6d 100644 --- a/src/common/espn_dates.py +++ b/src/common/espn_dates.py @@ -947,7 +947,8 @@ def get_espn_scoreboard( split as ESPN needs -- caches the result under the canonical key and returns it. ``max_age=0`` always fetches (and still caches, for other readers). Errors raise exactly as :func:`fetch_espn_scoreboard` does, - and nothing is cached then. ``session=None`` uses the fetch service's + and nothing is cached then; nor is an answer whose ``events`` is not a + list (ESPN has sent ``"events": null``), which is returned as it came. ``session=None`` uses the fetch service's pooled session for the ESPN host. """ key = espn_scoreboard_cache_key(sport, league, dates) @@ -969,5 +970,7 @@ def get_espn_scoreboard( # cache read above would have accepted. cache_max_age=None if max_age is None else max(0.0, float(max_age)), ) - store_espn_scoreboard_cache(cache_manager, key, data) + # Every reader of the shared key would inherit a malformed answer. + if isinstance(data, dict) and isinstance(data.get("events"), list): + store_espn_scoreboard_cache(cache_manager, key, data) return data diff --git a/src/common/sports_live_fetch.py b/src/common/sports_live_fetch.py new file mode 100644 index 00000000..b357c6de --- /dev/null +++ b/src/common/sports_live_fetch.py @@ -0,0 +1,200 @@ +"""The live poll's scoreboard request and the odds requests (sports family 9). + +The scoreboards' live fetch and odds code, reconciled in ledmatrix-plugins +(family 9) from seven and three bodies into one each, and copied here under +the existing names. All of it lives on the plugins' ``SportsCore``, so there +is one mixin, ``SportsLiveFetchMixin``: + +- ``_fetch_todays_games()``: today's scoreboard for the live poll, with + yesterday's while it can still hold a live game (``_needs_previous_day``), + as ``{"events": [...]}``; None when ESPN cannot be reached. +- ``_fetch_odds(game)``: queue one game for an odds request; returns at once. +- ``_drain_odds_queue()``: the odds worker that answers the queue. + +THE RULES (decided 2026-10-12) +------------------------------ +Live scores are always fresh. Every poll asks ESPN (``get_espn_scoreboard`` +with ``max_age=1``: only a copy another reader fetched within the last second, +the managers of one display tick, is reused) and leaves the answer under the +shared ``espn_scoreboard_cache_key`` for its dates, so odds-ticker and every +other reader of the same day share it. The plugins' old 30 s cache under +``_scoreboard_current`` is gone. College basketball is asked by +date like every other league; ESPN answered single dates for it on 2026-10-12 +(the old no-dates request cited a 404 that no longer happens). An answer whose +``events`` is not a list counts as no games. + +Odds never block ``update()``. ``_fetch_odds`` puts the game on the manager's +queue and starts one daemon worker if none is running; the worker asks the +odds manager for each queued game in turn and attaches the line to every +dict queued for that id, in place, under ``_games_lock`` (display() reads the +games under it), so a card can draw once without odds and gain them when +ESPN answers. One batch in flight per manager: a game queued while the worker +is busy joins its batch, so a slow endpoint never stacks threads or requests. +The odds manager bounds each request (core's ``BaseOddsManager``: 5 s, then a +60 s hold-off). A live game's line is cached ``live_odds_update_interval`` +(60 s), any other ``odds_update_interval`` (3600 s). Without ``show_odds``, a +game id or an odds manager nothing is asked (the last logged at debug). + +Before the reconcile, six plugins waited on a worker for 1.5 s (2 s for a +live game) per game and hockey and lacrosse asked on the calling thread: with +a 3 s endpoint, ``update()`` of four upcoming games took 6 s and 12 s, and in +the six the line appeared only at the next ``update()`` (an hour later for an +upcoming game). + +A new module rather than more methods on ``sports_fetch``, for the reason +``sports_helpers`` gives: a missing module fails at load, where the version +checks see it; a missing method fails mid-update. + +WHAT A HOST MUST PROVIDE +------------------------ +Derived by walking every ``self.`` the mixin reads; the host-contract +test in ``test/test_sports_live_fetch.py`` fails if a read is added without +being listed here. Every scoreboard ``SportsCore`` supplies all of them. + +- ``session``, ``headers``, ``cache_manager``, ``logger``, ``sport``, + ``league`` -- the scoreboard request. +- ``_needs_previous_day`` -- from ``SportsFetchMixin``. +- ``show_odds``, ``mode_config`` (the two odds intervals), ``sport_key`` (the + worker's thread name), ``_games_lock`` (a lock display() also holds). +- ``odds_manager``, read with ``getattr`` -- ``get_odds(sport=, league=, + event_id=, update_interval_seconds=)``. +- ``_odds_pending`` and ``_odds_worker_busy`` -- the queue and whether a + worker is running; created here on first use, under ``_games_lock``. + +The game dict's ``id`` and ``is_live`` are read; ``odds`` is written. + +BASE ORDER +---------- +No other mixin defines these names, so the position in the bases does not +change which body runs; a method on the plugin's own class still wins. The +mixin has no ``__init__``. +""" + +import threading +from datetime import datetime, timedelta +from typing import Any, Callable, Dict, List, Optional, Tuple + +import pytz +import requests + +from src.common.espn_dates import get_espn_scoreboard + + +class SportsLiveFetchMixin: + """``SportsCore``'s live fetch and odds. See module docstring.""" + + # The host contract, declared for type checking only. + session: Any + headers: Dict[str, str] + cache_manager: Any + logger: Any + sport: str + league: str + sport_key: str + show_odds: bool + mode_config: Dict[str, Any] + _games_lock: Any + _needs_previous_day: Callable[[datetime], bool] + odds_manager: Any + _odds_pending: Dict[Any, Tuple[List[Dict], Any]] + _odds_worker_busy: bool + + def _fetch_todays_games(self) -> Optional[Dict]: + """Today's scoreboard for the live poll, with yesterday's while it can + still hold a live game (``_needs_previous_day``).""" + # ESPN anchors its calendar to Eastern US time. + now = datetime.now(pytz.timezone("America/New_York")) + today = now.strftime("%Y%m%d") + dates = ( + f"{(now - timedelta(days=1)).strftime('%Y%m%d')}-{today}" + if self._needs_previous_day(now) else today + ) + try: + # Asked on every poll and left under core's shared key for this + # scoreboard and date, so odds-ticker and every other reader of + # the same day reuse it. max_age=1: a copy another reader fetched + # within the last second (the managers of one display tick) is the + # same answer; anything older is asked again. + data = get_espn_scoreboard( + self.session, self.sport, self.league, dates, + cache_manager=self.cache_manager, max_age=1, + headers=self.headers, timeout=10, logger=self.logger, + ) + except requests.exceptions.RequestException as e: + self.logger.error( + f"API error fetching today's games for {self.sport} - {self.league}: {e}" + ) + return None + events = data.get("events") if isinstance(data, dict) else None + if not isinstance(events, list): + events = [] # ESPN can answer "events": null + self.logger.info( + f"Fetched {len(events)} games for {dates} for {self.sport} - {self.league}" + ) + return {"events": events} + + def _fetch_odds(self, game: Dict) -> None: + """Queue ``game`` for an odds request; never waits for the answer. + + One daemon worker per manager asks the odds manager for each queued + game in turn and attaches the line to the game dict in place, under + ``_games_lock`` (display() reads the games under it), so a card can + draw once without odds and gain them when ESPN answers. A game queued + while the worker is busy joins its batch, and every dict queued for one + id (update() rebuilds them; the list on screen may keep older ones) + gets the one answer, so a slow endpoint never stacks threads or + requests. The odds manager bounds each request (5 s, then a 60 s + hold-off). + """ + if not self.show_odds or game.get("id") is None: + return + if not getattr(self, "odds_manager", None): + self.logger.debug(f"No odds manager; no odds for game {game['id']}") + return + interval = ( + self.mode_config.get("live_odds_update_interval", 60) + if game.get("is_live", False) + else self.mode_config.get("odds_update_interval", 3600) + ) + with self._games_lock: + pending = getattr(self, "_odds_pending", None) + if pending is None: + pending = self._odds_pending = {} + games, _ = pending.get(game["id"], ([], None)) + if not any(queued is game for queued in games): + games.append(game) + pending[game["id"]] = (games, interval) + if getattr(self, "_odds_worker_busy", False): + return + self._odds_worker_busy = True + try: + threading.Thread(target=self._drain_odds_queue, daemon=True, + name=f"{self.sport_key}-odds").start() + except RuntimeError as e: # no thread to be had; try again next update + with self._games_lock: + self._odds_worker_busy = False + self.logger.debug(f"Could not start the odds worker: {e}") + + def _drain_odds_queue(self) -> None: + """The odds worker: ask for each queued game until none is left.""" + while True: + with self._games_lock: + if not self._odds_pending: + self._odds_worker_busy = False + return + game_id = next(iter(self._odds_pending)) + games, interval = self._odds_pending.pop(game_id) + try: + odds = self.odds_manager.get_odds( + sport=self.sport, + league=self.league, + event_id=game_id, + update_interval_seconds=interval, + ) + except Exception as e: + self.logger.debug(f"Odds fetch failed for game {game_id}: {e}") + continue + if odds: + with self._games_lock: + for game in games: + game["odds"] = odds diff --git a/test/test_espn_scoreboard_cache.py b/test/test_espn_scoreboard_cache.py index 79d92869..05b0bbf2 100644 --- a/test/test_espn_scoreboard_cache.py +++ b/test/test_espn_scoreboard_cache.py @@ -392,6 +392,15 @@ class TestGetEspnScoreboard: "20261004", cache_manager=cache) assert cache.records == {} + @pytest.mark.parametrize("payload", [{"events": None}, {}, {"events": "x"}, []]) + def test_an_answer_without_an_events_list_is_returned_not_cached(self, service, payload): + espn = Espn() + espn.get = lambda url, **kwargs: _response(payload, url, None) + cache = RecordCache() + assert get_espn_scoreboard(espn, "football", "nfl", "20261004", + cache_manager=cache) == payload + assert cache.records == {} + def test_no_cache_manager_just_fetches(self, service): espn = Espn() get_espn_scoreboard(espn, "football", "nfl", "20261004") diff --git a/test/test_sports_live_fetch.py b/test/test_sports_live_fetch.py new file mode 100644 index 00000000..e60531ce --- /dev/null +++ b/test/test_sports_live_fetch.py @@ -0,0 +1,399 @@ +"""src.common.sports_live_fetch: the live poll's request and the odds queue. + +The cases follow ledmatrix-plugins' ``scripts/test_live_fetch_odds.py`` (the +tables the family 9 reconcile was checked against); the expected values are +its columns, which agree across all nine scoreboards. The host is the mixin +over core's ``SportsFetchMixin`` (``_needs_previous_day``), with a session +answering like ESPN did on 2026-10-12 (a single date is 200, a range 400) and +a cache that ages entries on a test clock. Nothing reaches the network. The +odds endpoint is held on an event, so "update() never waits" is shown by the +call returning while the request is still open. +""" + +import ast +import json +import logging +import threading +import time +from datetime import datetime +from pathlib import Path + +import pytest +import requests + +from src.common import espn_dates, sports_live_fetch +from src.common import fetch_service as fs +from src.common.fetch_service import FetchService +from src.common.sports_fetch import SportsFetchMixin +from src.common.sports_live_fetch import SportsLiveFetchMixin + +LOG = logging.getLogger("test_sports_live_fetch") +SITE = "https://site.api.espn.com/apis/site/v2/sports/" + + +def _response(url, answer): + resp = requests.Response() + resp.url = url + resp.status_code = answer if isinstance(answer, int) else 200 + resp._content = answer if isinstance(answer, bytes) else json.dumps( + {} if isinstance(answer, int) else answer).encode() + return resp + + +class Espn(requests.Session): + """Two games a day for a single date; a range is 400, as ESPN answers.""" + + def __init__(self, answer=None): + super().__init__() + self.answer, self.asked = answer, [] + + def get(self, url, params=None, **_kwargs): + dates = (params or {}).get("dates") + self.asked.append((url[len(SITE):], dates)) + if dates and "-" in dates: + return _response(url, 400) + if isinstance(self.answer, type): + raise self.answer("refused by the test") + if self.answer is not None: + return _response(url, self.answer) + return _response(url, {"events": [{"id": dates[4:] + s} for s in "ab"]}) + + +class Cache: + """``get(key, max_age)`` against entries stamped on ``clock``.""" + + def __init__(self, clock): + self.clock, self.store, self.reads = clock, {}, [] + + def get(self, key, max_age=None): + self.reads.append((key, max_age)) + if key not in self.store: + return None + data, stamp = self.store[key] + return data if max_age is None or self.clock() - stamp <= max_age else None + + def set(self, key, data, ttl=None): + self.store[key] = (data, self.clock()) + + +class Clock: + def __init__(self): + self.now = 100_000.0 + + def __call__(self): + return self.now + + +class Frozen(datetime): + at = datetime(2026, 10, 12, 10, 0) + + @classmethod + def now(cls, tz=None): + return tz.localize(cls.at) if tz is not None else cls.at + + +class Host(SportsLiveFetchMixin, SportsFetchMixin): + """A scoreboard SportsCore as far as the family reaches.""" + + sport, league, sport_key = "basketball", "nba", "nba" + + def __init__(self, session=None, cache=None, odds=None, show_odds=True): + self.session, self.headers, self.logger = session or Espn(), {}, LOG + self.cache_manager = cache + self.odds_manager, self.show_odds = odds, show_odds + self.mode_config = {} + self._games_lock = threading.RLock() + + +@pytest.fixture(autouse=True) +def offline(monkeypatch): + clock = Clock() + monkeypatch.setattr(fs, "_service", FetchService({"rate_limits": {}})) + monkeypatch.setattr(espn_dates, "_ranges_rejected_until", 0.0) + monkeypatch.setattr(sports_live_fetch, "datetime", Frozen) + monkeypatch.setattr(time, "time", clock) + Frozen.at = datetime(2026, 10, 12, 10, 0) + return clock + + +# --------------------------------------------------------------------------- +# The live poll's request +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("hour, asked, key, ids", [ + (10, [("20261012",)], "espn_scoreboard_basketball_nba_20261012", ["1012a", "1012b"]), + (2, [("20261011-20261012",), ("20261011",), ("20261012",)], + "espn_scoreboard_basketball_nba_20261011-20261012", + ["1011a", "1011b", "1012a", "1012b"]), +]) +def test_what_the_poll_asks_and_keeps(offline, hour, asked, key, ids): + Frozen.at = datetime(2026, 10, 12, hour, 0) + espn, cache = Espn(), Cache(offline) + data = Host(espn, cache)._fetch_todays_games() + assert [(d,) for _, d in espn.asked] == asked + assert {path for path, _ in espn.asked} == {"basketball/nba/scoreboard"} + assert cache.reads == [(key, 1)] + assert list(cache.store) == [key] + assert [e["id"] for e in data["events"]] == ids + + +def test_college_basketball_is_asked_by_date(offline): + espn, host = Espn(), Host() + host.session, host.league = espn, "mens-college-basketball" + host._fetch_todays_games() + assert espn.asked == [("basketball/mens-college-basketball/scoreboard", "20261012")] + + +@pytest.mark.parametrize("seed_age, later, requests_made", [ + (None, 20, [1, 1]), # every poll asks ESPN; nothing is held 30 s + (None, 0.5, [1, 0]), # a copy from this second is the same answer + (0, None, [0]), # another reader's copy from this second + (2, None, [1]), # ... and one two seconds old is asked again +]) +def test_every_poll_is_fresh(offline, seed_age, later, requests_made): + espn, cache = Espn(), Cache(offline) + if seed_age is not None: + cache.store["espn_scoreboard_basketball_nba_20261012"] = ( + {"events": [{"id": "seed"}]}, offline.now - seed_age) + host, made = Host(espn, cache), [] + for step in ([0] if later is None else [0, later]): + offline.now += step + before = len(espn.asked) + host._fetch_todays_games() + made.append(len(espn.asked) - before) + assert made == requests_made + + +def test_the_old_per_plugin_key_is_not_read(offline): + cache = Cache(offline) + cache.store["nba_scoreboard_current"] = ({"events": [{"id": "old"}]}, offline.now) + data = Host(Espn(), cache)._fetch_todays_games() + assert [e["id"] for e in data["events"]] == ["1012a", "1012b"] + assert "nba_scoreboard_current" not in {k for k, _ in cache.reads} + + +@pytest.mark.parametrize("answer, result, cached", [ + ({}, {"events": []}, False), + ({"events": None}, {"events": []}, False), + ({"events": []}, {"events": []}, True), + (404, None, False), + (500, None, False), + (requests.ConnectionError, None, False), + (requests.Timeout, None, False), + (b"", None, False), +]) +def test_what_a_failed_or_malformed_answer_becomes(offline, answer, result, cached): + cache = Cache(offline) + assert Host(Espn(answer), cache)._fetch_todays_games() == result + assert bool(cache.store) is cached + + +# --------------------------------------------------------------------------- +# Odds +# --------------------------------------------------------------------------- + +class Odds: + """An odds endpoint held open until ``release()``, counting what is in + flight and how many odds workers are alive while it answers.""" + + def __init__(self, answer="odds", held=True): + self.answer, self.calls, self.gate = answer, [], threading.Event() + if not held: + self.gate.set() + self.lock, self.active, self.most_active, self.most_workers = threading.Lock(), 0, 0, 0 + + def get_odds(self, sport, league, event_id, update_interval_seconds=None): + workers = sum(t.name == "nba-odds" and t.is_alive() for t in threading.enumerate()) + with self.lock: + self.calls.append((event_id, update_interval_seconds)) + self.active += 1 + self.most_active = max(self.most_active, self.active) + self.most_workers = max(self.most_workers, workers) + self.gate.wait(10) + with self.lock: + self.active -= 1 + if isinstance(self.answer, type): + raise self.answer("odds endpoint failed") + return {"details": f"{event_id} -3.5"} if self.answer else None + + def release(self): + self.gate.set() + + +def settle(): + for thread in threading.enumerate(): + if thread.name == "nba-odds": + thread.join(10) + + +def game(gid="g1", **flags): + return {"id": gid, **flags} + + +@pytest.mark.parametrize("flags, interval", [({"is_live": True}, 60), ({"is_upcoming": True}, 3600), + ({"is_final": True}, 3600)]) +def test_the_call_never_waits_and_the_line_lands_later(flags, interval): + odds = Odds() + host, g = Host(odds=odds), game(**flags) + host._fetch_odds(g) # returns while the request is held open + assert "odds" not in g + odds.release() + settle() + assert g["odds"] == {"details": "g1 -3.5"} + assert odds.calls == [("g1", interval)] + + +def test_the_intervals_come_from_the_mode_config(): + odds = Odds(held=False) + host = Host(odds=odds) + host.mode_config = {"live_odds_update_interval": 30, "odds_update_interval": 900} + host._fetch_odds(game("l", is_live=True)) + settle() + host._fetch_odds(game("u")) + settle() + assert odds.calls == [("l", 30), ("u", 900)] + + +@pytest.mark.parametrize("answer", [None, RuntimeError]) +def test_no_line_or_a_failed_request_leaves_the_game_bare(caplog, answer): + odds = Odds(answer, held=False) + host, g = Host(odds=odds), game() + with caplog.at_level(logging.DEBUG, logger=LOG.name): + host._fetch_odds(g) + settle() + assert "odds" not in g and host._odds_worker_busy is False + assert [r for r in caplog.records if r.levelno >= logging.WARNING] == [] + + +@pytest.mark.parametrize("host", [ + Host(odds=Odds(held=False), show_odds=False), + Host(odds=None), +], ids=["show_odds off", "no odds manager"]) +def test_nothing_is_asked(caplog, host): + with caplog.at_level(logging.DEBUG, logger=LOG.name): + host._fetch_odds(game()) + assert not [t for t in threading.enumerate() if t.name == "nba-odds"] + assert [r.levelname for r in caplog.records] in ([], ["DEBUG"]) + + +def test_a_game_without_an_id_is_not_asked(): + odds = Odds(held=False) + Host(odds=odds)._fetch_odds({"is_live": True}) + settle() + assert odds.calls == [] + + +def test_one_batch_per_manager_and_every_dict_gets_the_answer(): + odds = Odds() + host = Host(odds=odds) + first, again, other = game("g1"), game("g1"), game("g2") + with host._games_lock: # queued before the worker takes one + for g in (first, again, other): # update() rebuilds the dicts + host._fetch_odds(g) + odds.release() + settle() + assert odds.calls == [("g1", 3600), ("g2", 3600)] + assert odds.most_workers == 1 + assert first["odds"] == again["odds"] == {"details": "g1 -3.5"} + + +def test_a_thread_that_will_not_start_is_tried_again(monkeypatch): + class NoThread: + def __init__(self, *args, **kwargs): + pass + + def start(self): + raise RuntimeError("can't start new thread") + + odds = Odds(held=False) + host = Host(odds=odds) + monkeypatch.setattr(sports_live_fetch.threading, "Thread", NoThread) + host._fetch_odds(game("g1")) + assert host._odds_worker_busy is False + monkeypatch.undo() + host._fetch_odds(game("g2")) + settle() + assert [c[0] for c in odds.calls] == ["g1", "g2"] + + +def test_update_and_display_on_two_threads(): + """Two updaters queueing fresh dicts while a reader walks the games under + the lock, as display() does: no exception, one worker, one request at a + time, and every dict ends with its line.""" + odds = Odds(held=False) + odds.get_odds = _slowed(odds.get_odds) + host, shown, raised = Host(odds=odds), [], [] + + def updater(): + try: + for n in range(30): + g = game(f"g{n % 4}") + with host._games_lock: + shown.append(g) + host._fetch_odds(g) + except Exception as exc: # noqa: BLE001 + raised.append(exc) + + def display(): + try: + for _ in range(300): + with host._games_lock: + [g.get("odds") for g in shown] + except Exception as exc: # noqa: BLE001 + raised.append(exc) + + threads = [threading.Thread(target=updater) for _ in range(2)] + threads.append(threading.Thread(target=display)) + for t in threads: + t.start() + for t in threads: + t.join(30) + settle() + assert raised == [] + assert odds.most_active == 1 and odds.most_workers == 1 + assert all(g.get("odds") for g in shown) + + +def _slowed(get_odds): + def slow(**kwargs): + time.sleep(0.005) + return get_odds(**kwargs) + return slow + + +# --------------------------------------------------------------------------- +# Carrier and host contract +# --------------------------------------------------------------------------- + +MEMBERS = ["_drain_odds_queue", "_fetch_odds", "_fetch_todays_games"] + + +def _self_reads(): + tree = ast.parse(Path(sports_live_fetch.__file__).read_text(encoding="utf-8")) + reads = {node.attr for node in ast.walk(tree) + if isinstance(node, ast.Attribute) and isinstance(node.ctx, ast.Load) + and isinstance(node.value, ast.Name) and node.value.id == "self"} + gets = {node.args[1].value for node in ast.walk(tree) + if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "getattr" + and len(node.args) >= 2 and isinstance(node.args[1], ast.Constant)} + return reads | gets + + +class TestHostContract: + def test_the_mixin_carries_only_the_family(self): + assert sorted(n for n in vars(SportsLiveFetchMixin) if not n.startswith("__")) == MEMBERS + + def test_every_host_read_is_documented(self): + undocumented = sorted(n for n in _self_reads() - set(MEMBERS) + if f"``{n}``" not in sports_live_fetch.__doc__) + assert undocumented == [], f"read but not in the host contract: {undocumented}" + + def test_no_other_shared_mixin_defines_these(self): + from src.common import (sports_display_rules, sports_favorites, sports_helpers, + sports_rankings, sports_rotation, sports_shared) + others = [sports_shared.SportsCoreSharedMixin, sports_shared.SportsRecentSharedMixin, + sports_shared.SportsLiveSharedMixin, sports_helpers.SportsHelpersMixin, + sports_favorites.SportsFavoritesMixin, sports_rotation.SportsRotationMixin, + sports_rankings.SportsRankingsMixin, SportsFetchMixin, + sports_display_rules.SportsGameRulesMixin] + assert [m for m in MEMBERS if any(m in vars(o) for o in others)] == [] diff --git a/test/test_sports_live_fetch_parity.py b/test/test_sports_live_fetch_parity.py new file mode 100644 index 00000000..aab1b1b2 --- /dev/null +++ b/test/test_sports_live_fetch_parity.py @@ -0,0 +1,104 @@ +"""sports_live_fetch still matches every plugin copy. + +``src.common.sports_live_fetch`` was copied from the scoreboards once family 9 +had made each method one body in all nine ``SportsCore`` classes: +``_fetch_todays_games``, ``_fetch_odds`` and ``_drain_odds_queue``. The plugins delete their copies once they floor on the release that +ships this module. Until each has, a copy that changes on its own is a fix one +side has and the other lacks. + +Point LEDMATRIX_PLUGINS at a ledmatrix-plugins checkout and each method is +compared with every plugin copy using ``scripts/sports_drift_report.py``'s own +normalisation (the AST with docstrings and annotations dropped), plus the +decorators. A copy that is gone counts as adopted when the plugin's ``sports.py`` names the module. +Without the variable this skips: core CI has no plugins checkout. +""" + +import ast +import importlib.util +import os +from pathlib import Path + +import pytest + +from src.common import sports_live_fetch + +REPO = Path(__file__).resolve().parents[1] +SPORTS = ("afl", "baseball", "basketball", "football", "hockey", "lacrosse", + "nrl", "soccer", "ufc") +CARRIER, MIXIN = "SportsCore", "SportsLiveFetchMixin" +METHODS = ("_fetch_todays_games", "_fetch_odds", "_drain_odds_queue") + + +def _drift_report(): + """scripts/sports_drift_report.py, loaded by path (scripts/ is no package).""" + spec = importlib.util.spec_from_file_location( + "sports_drift_report", REPO / "scripts" / "sports_drift_report.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +DRIFT = _drift_report() + + +def _plugins_root(): + root = DRIFT.resolve_plugins_dir(os.environ.get("LEDMATRIX_PLUGINS")) + if root is None: + pytest.skip("set LEDMATRIX_PLUGINS to a ledmatrix-plugins checkout to " + "compare this module against the plugin copies") + return root + + +def _class(tree, name): + return next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == name) + + +def _method(cls, name): + return next((n for n in cls.body + if isinstance(n, ast.FunctionDef) and n.name == name), None) + + + +def _fingerprint(node): + return (DRIFT._digest(node, DRIFT._Canonical()), + tuple(ast.unparse(d) for d in node.decorator_list)) + + +def _plugin_tree(root, sport): + source = (root / f"{sport}-scoreboard" / "sports.py").read_text(encoding="utf-8") + return source, ast.parse(source) + + +OURS = _class(ast.parse(Path(sports_live_fetch.__file__).read_text(encoding="utf-8")), MIXIN) + + +@pytest.mark.parametrize("sport, name", [(s, n) for s in SPORTS for n in METHODS]) +def test_every_remaining_plugin_copy_matches(sport, name): + source, tree = _plugin_tree(_plugins_root(), sport) + copy = _method(_class(tree, CARRIER), name) + if copy is None: + assert sports_live_fetch.__name__ in source, ( + f"{sport}: no {name} on {CARRIER} and no {sports_live_fetch.__name__} import") + else: + assert _fingerprint(copy) == _fingerprint(_method(OURS, name)), ( + f"{CARRIER}.{name} in {sport} differs from sports_live_fetch. " + f"Port the change to both, or stop treating it as shared.") + + + +@pytest.mark.parametrize("sport", SPORTS) +def test_no_other_plugin_class_carries_a_copy(sport): + """A copy on another class would shadow the shared one for that class.""" + _, tree = _plugin_tree(_plugins_root(), sport) + strays = [f"{node.name}.{name}" for node in tree.body if isinstance(node, ast.ClassDef) + for name in METHODS if node.name != CARRIER and _method(node, name) is not None] + assert strays == [] + + +def test_the_drift_report_calls_them_identical(): + families = DRIFT.build(_plugins_root(), ("sports.py",)) + rows = {(r["file"], r["family"]): r + for r in (DRIFT.summarise(k, v) for k, v in families.items())} + for name in METHODS: + row = rows.get(("sports.py", name)) + assert row is None or row["worst_class_variants"] == 1, name