From 3bfb4a139ef91c9af20d64855dbf600dbfe87e33 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:45:04 -0400 Subject: [PATCH] feat(common): sports_live_fetch -- the reconciled live fetch and odds (sports family 9) New hardware-free module src/common/sports_live_fetch.py, copied from ledmatrix-plugins claude/family9-reconcile once the nine scoreboards made _fetch_todays_games (seven bodies when measured) and _fetch_odds (three) one body each. Both live on the plugins' SportsCore, so one mixin, SportsLiveFetchMixin, carries exactly those two plus the odds worker _drain_odds_queue; no manager gains a method it lacks today. Owner decisions (2026-10-12): - Live scores are always fresh: every poll asks ESPN through get_espn_scoreboard and leaves the answer under the shared espn_scoreboard_cache_key for its dates (max_age=1: only a copy another reader fetched within the last second is reused). The plugins' 30 s _scoreboard_current cache is gone. - College basketball is asked by date like every other league (ESPN answered single dates for both college leagues on 2026-10-12). - Odds never block update(): one daemon worker per manager answers a queue and attaches each line in place under _games_lock; one batch in flight per manager, games queued meanwhile join it, every dict queued for an id gets the one answer. show_odds, the two odds intervals and a missing odds manager (debug, not ERROR) are respected. - An answer whose events is not a list is no games. Fix: get_espn_scoreboard() no longer caches an answer whose events is not a list ("events": null), which every reader of the shared key inherited; it is still returned to the caller. - test/test_sports_live_fetch.py: the plugins' pinned cases (requests, dates and keys at 10:00 and 02:00 Eastern, freshness, malformed and failed answers, odds that never block and land later, the intervals, one batch per manager, a worker that cannot start, update and display on two threads), plus the host contract. Mutation-checked (11 mutants, all caught). - test/test_sports_live_fetch_parity.py: with LEDMATRIX_PLUGINS, compares each body with every plugin copy. Passes against claude/family9-reconcile; fails against the pre-reconcile tree. - test/test_espn_scoreboard_cache.py: the null-events regression. - mypy ratchet, src/common/README.md, CHANGELOG (Unreleased: New modules, Fixes), docs/SPORTS_UNIFICATION.md family 9 status and decisions. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 20 ++ docs/SPORTS_UNIFICATION.md | 70 ++++- mypy-clean.txt | 1 + src/common/README.md | 14 + src/common/espn_dates.py | 7 +- src/common/sports_live_fetch.py | 200 +++++++++++++ test/test_espn_scoreboard_cache.py | 9 + test/test_sports_live_fetch.py | 399 ++++++++++++++++++++++++++ test/test_sports_live_fetch_parity.py | 104 +++++++ 9 files changed, 812 insertions(+), 12 deletions(-) create mode 100644 src/common/sports_live_fetch.py create mode 100644 test/test_sports_live_fetch.py create mode 100644 test/test_sports_live_fetch_parity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a709790..1ca490de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,28 @@ accepts both, but the store flags the old spelling as deprecated ## Unreleased +### New modules + +- `src/common/sports_live_fetch.py` -- sports consolidation family 9, once the + plugins made `_fetch_todays_games` (seven bodies) and `_fetch_odds` (three) + one each: `SportsLiveFetchMixin` (`SportsCore`), with the odds worker + `_drain_odds_queue` beside them. The live poll asks ESPN on every poll + (a copy another reader fetched within the last second is reused) and + leaves the answer under the shared `espn_scoreboard_cache_key` for its + dates; college basketball is asked by date like every other league; an + answer whose `events` is not a list counts as no games. Odds never block + `update()`: one daemon worker per manager answers a queue and attaches each + line in place under `_games_lock`, one batch in flight at a time. A plugin + may inherit the mixin once it floors on the release that ships it, and + deletes its copies then. + ### Fixes +- `get_espn_scoreboard()` no longer caches an answer whose `events` is not a + list (ESPN has sent `"events": null`): every reader of the shared scoreboard + key -- odds-ticker, the other scoreboards -- inherited it until it aged out. + The answer is still returned to the caller as it came. + - `SportsRankingsMixin` (#789's deferred review points, now that it is the only copy): a poll entry whose `current` cannot be converted to an integer (`None`, a non-numeric string) is skipped instead of throwing diff --git a/docs/SPORTS_UNIFICATION.md b/docs/SPORTS_UNIFICATION.md index d89d911b..b7819ade 100644 --- a/docs/SPORTS_UNIFICATION.md +++ b/docs/SPORTS_UNIFICATION.md @@ -415,6 +415,45 @@ its id table too); the two differ only for a poll whose blocks carry ids and no abbreviations. The data sources' `fetch_standings` (held with `data_sources.py`) is still four bodies; only its endpoint order changed. +### Family 9: live fetch and odds (reconciled; promoted, awaiting release) + +ledmatrix-plugins `scripts/test_live_fetch_odds.py` (#649) pinned 44 rows +across the nine plugins first, every ESPN request answered offline the way +ESPN answered on 2026-10-12 (a single date 200, a range 400): the requests, +cache keys and TTLs of `_fetch_todays_games` per league and Eastern time, +the live cache over time, empty, null and failed answers, `_fetch_odds` per +game kind and odds endpoint on a virtual clock (which thread asks, how long +the call waits, when the line lands), the real `update()` of every screen +with a 3 s odds endpoint, and the league overrides (MiLB, HockeyTech, ufc's +`MMALive`, which stay per-sport). Measuring found seven bodies of +`_fetch_todays_games`, not five: the fetch service's shared key had split +the old groups. The reconcile (ledmatrix-plugins `claude/family9-reconcile`) +made `_fetch_todays_games` and `_fetch_odds` one body each on `SportsCore`, +with the odds worker `_drain_odds_queue` beside them, and added a test that +runs `update()` and `display()` on two threads against a slow endpoint. +Every changed cell is one of the decisions below; the harness renders were +byte-identical once the four fixtures that seeded the old live key seed the +shared one. `src/common/sports_live_fetch.py` holds the bodies in one mixin, +`SportsLiveFetchMixin`; `test/test_sports_live_fetch_parity.py` compares them +with the plugin copies. + +Measured `update()` time with a 3 s odds endpoint and four games, before and +after: Upcoming 6 s (afl, baseball, football, nrl, soccer, ufc), 12 s +(hockey, lacrosse), 0 s (basketball) to 0 s everywhere; Recent 6 s / 12 s to +0 s; Live's first pass 8 s / 12 s and next poll 4 s / 6 s to 0 s. The line +now lands when ESPN answers (3 s here) instead of at the next `update()` +(15 s for a live game, an hour for an upcoming or recent one) in the six. + +Left for later: ufc's MMA Recent manager never asks for odds and its Upcoming +manager asks for every fight in the window before trimming to the headliners +(both in `mma.py`'s `update()` overrides, family 13); basketball's NCAA +Recent/Upcoming fallback still caches the undated scoreboard under +`_scoreboard_current` with `limit=1000`, which ESPN truncates (held +with the data fetching); `_attach_odds_to_rotated_games` still starts its own +thread per rotation and can route through `_fetch_odds` once every plugin +inherits this mixin; and ESPN lists only some college basketball games +without `groups=50` (7 of 21 on 2026-03-14), dated or not. + ### Why the method changes Byte-identical promotion has nearly run dry. Measured on ledmatrix-plugins @@ -503,7 +542,7 @@ release. | 6 | Favourite matching | `_is_favorite_game` (7 across three classes), `_select_games_for_display` (2: nrl), `_select_recent_games_for_display` (3) | Everything that asks "is this a favourite" goes through the 3.5.0 `_favorite_key` seam. Reconciled to one body each and promoted as `sports_favorites`; shipped in 3.8.2 and adopted. See [Family 6](#family-6-favourite-matching-done-core-382-adopted) | | 7 | Other-games rotation | `_by_importance`, `_other_games_window`, `_advance_other_games_if_due` (2 each: football), `_rotate_other_games_on_display` (2: ufc), with `_attach_odds_to_rotated_games` (3; ufc had none) | One outlier each; football carried two fixes the other eight lacked. Reconciled to one body each, on a `_rankings_loaded` seam, and promoted as `sports_rotation`; done: core 3.8.4, adopted (ledmatrix-plugins #643). See [Family 7](#family-7-the-other-games-rotation-done-core-384-adopted) | | 8 | Rankings | `_fetch_team_rankings` (3), `_choose_poll` (3), `_load_division_team_ids`, `_passes_other_filters`, `_best_rank`, `_is_ranked_game` (2 each: football) | Needs 7; the rank badge and the "ranked only" filter read it. Reconciled to one body each and promoted as `sports_rankings`; done: core 3.8.5, adopted (ledmatrix-plugins #648). See [Family 8](#family-8-rankings-done-core-385-adopted) | -| 9 | Live fetch and odds | `_fetch_todays_games` (5), `_fetch_odds` (3), `_attach_odds_to_rotated_games` (3) | The prerequisite for one shared ESPN poller across plugins | +| 9 | Live fetch and odds | `_fetch_todays_games` (7 when measured), `_fetch_odds` (3); `_attach_odds_to_rotated_games` went with 7 | The prerequisite for one shared ESPN poller across plugins. Reconciled to one body each and promoted as `sports_live_fetch` (next release). See [Family 9](#family-9-live-fetch-and-odds-reconciled-promoted-awaiting-release) | | 10 | View model | `_extract_game_details_common` (9 of 9) | Every renderer reads it; its keys are additive-only, so reconcile to the superset and leave sport extras in `_extract_game_details` | | 11 | Switch scorebug | `_load_fonts` (4), `_load_custom_font_from_element_config` (6), `_get_layout_offset` (5), `_fit_score_font` (2: football), `_load_and_resize_logo` (9), `_draw_dynamic_odds` (9), then `_draw_scorebug_layout` (20: Upcoming 9, Recent 8, Live 2, ufc's Core 1) | Needs 10 and the twin decisions below. Several releases: fonts and offsets, logos, odds, then one mode's layout per release | | 12 | Scroll/Vegas card | `game_renderer.py`: `render_game_card` (6), `_load_and_resize_logo` (8), `_load_custom_font`, `preload_logos`, `__init__` (7 each), `_draw_records_or_rankings` (6), `_load_fonts`, `_draw_dynamic_odds`, `_draw_live_game_status`, `_draw_recent_game_status`, `_get_team_display_text` (5 each), `_draw_text_with_outline` (2: football) | Same decisions as 11; sequence after the scroll-performance work on pre-rendered strips lands | @@ -598,15 +637,26 @@ suspected behaviour that needs a payload or a rig to confirm first. seasons. A block without a readable `endDate` counts as current. (f) baseball's MLB, MiLB and NCAA Baseball managers drop their class-level shared copy of the table (none of those leagues publishes a poll). -- **9, live fetch and odds.** (a) afl, basketball, nrl and soccer cache the - live scoreboard for 30 s under `_scoreboard_current`; the others - do not (the harness fixtures seed that key, so the change shows up there). - (b) basketball fetches college games with no `dates` parameter, citing a - 404 (*verify* now that `espn_dates` handles ranges). (c) odds are fetched - three ways: blocking (hockey, lacrosse), a thread waited on for 1.5–2 s - (six plugins), or fire-and-forget for upcoming games (basketball). This - sets how long `update()` takes and when an odds line appears. (d) nrl - guards on a missing odds manager; port it. +- **9, live fetch and odds. Decided 2026-10-12, reconciled.** (a) Live + scores are always fresh: every poll asks ESPN and leaves the answer under + the shared `espn_scoreboard_cache_key` for its dates (`max_age=1`: only a + copy another reader fetched within the last second, the managers of one + display tick, is reused). afl, basketball, nrl and soccer had held it 30 s + under `_scoreboard_current`, so with the 15 s default poll every + other poll repeated the last one; lacrosse and ufc now share their copy. + (b) College basketball is asked by date like every other league: ESPN + answered single dates for both college leagues on 2026-10-12 (the 404 no + longer happens), so the lookback now reaches last night's late games there + too. `groups=50` is a separate question (left above). (c) Odds never block + `update()`: one daemon worker per manager answers a queue and attaches each + line in place under `_games_lock`; one batch in flight per manager, games + queued meanwhile join it. A card can draw once without odds and gain them; + a pre-rendered scroll strip gains them when it is next rebuilt. (d) The + missing-odds-manager guard (nrl's, already in `_attach_odds_to_rotated_games` + since family 7) is in `_fetch_odds` too, logged at debug, not ERROR. Fix + ported: an answer with `"events": null` is no games instead of a + `TypeError` out of the fetch (all nine), and `get_espn_scoreboard` no longer + caches it under the shared key. - **10, view model.** Per key, whether every sport emits it. Additive only: no key is renamed or removed. - **11 and 12, the scorebug and the card.** The pinned divergences in diff --git a/mypy-clean.txt b/mypy-clean.txt index c88ea3a5..a3f8920f 100644 --- a/mypy-clean.txt +++ b/mypy-clean.txt @@ -39,6 +39,7 @@ src/common/sports_favorites.py src/common/sports_fetch.py src/common/sports_font_path.py src/common/sports_game_over.py +src/common/sports_live_fetch.py src/common/sports_live_scroll.py src/common/sports_plugin_host.py src/common/sports_rankings.py diff --git a/src/common/README.md b/src/common/README.md index 6e8e5882..4168d141 100644 --- a/src/common/README.md +++ b/src/common/README.md @@ -51,6 +51,7 @@ Rules for the package: | [`sports_game_over`](#sports_game_over) | Whether a game ESPN still lists as live has ended | Yes (scoreboards) | 3.8.1 | | [`sports_game_renderer`](#sports_game_renderer) | Scoreboard scroll/Vegas card geometry | Yes (scoreboards) | 3.3.0 | | [`sports_helpers`](#sports_helpers) | Small helpers every scoreboard `sports.py` copies | Yes (scoreboards) | 3.5.0 | +| [`sports_live_fetch`](#sports_live_fetch) | The live poll's scoreboard request, and odds off the update path | Yes (scoreboards) | next release | | [`sports_live_scroll`](#sports_live_scroll) | Rebuild a live scroll strip mid-cycle without moving it | Yes (scoreboards) | 3.8.0 | | [`sports_plugin_host`](#sports_plugin_host) | Helpers of a scoreboard's plugin class (`manager.py`) | Yes (scoreboards) | 3.8.0 | | [`sports_rankings`](#sports_rankings) | Which teams are ranked, in which poll, and the "ranked" filter | Yes (scoreboards) | 3.8.5 | @@ -370,6 +371,19 @@ shape), `_dispatch_switch_refresh()` (a manager refresh on a daemon thread, so `display()` never waits on the network), `get_vegas_content_type()` and small dynamic-duration helpers. List it before `BasePlugin`. +### sports_live_fetch + +[`sports_live_fetch.py`](sports_live_fetch.py). Sports family 9: +`SportsLiveFetchMixin` (`SportsCore`). `_fetch_todays_games` asks ESPN for +today's scoreboard (and yesterday's while `_needs_previous_day()` says it can +still hold a live game) on every live poll through `get_espn_scoreboard`, and +leaves it under the shared `espn_scoreboard_cache_key` for odds-ticker and the +other scoreboards; an answer without an `events` list is no games. +`_fetch_odds` queues a game and returns at once; one daemon worker per manager +(`_drain_odds_queue`) asks the odds manager for each queued game and attaches +the line in place under `_games_lock`, so a card can draw once without odds and +gain them. Games queued while the worker is busy join its batch. + ### sports_rankings [`sports_rankings.py`](sports_rankings.py). Sports family 8: diff --git a/src/common/espn_dates.py b/src/common/espn_dates.py index 5c8080a6..fbb81e6d 100644 --- a/src/common/espn_dates.py +++ b/src/common/espn_dates.py @@ -947,7 +947,8 @@ def get_espn_scoreboard( split as ESPN needs -- caches the result under the canonical key and returns it. ``max_age=0`` always fetches (and still caches, for other readers). Errors raise exactly as :func:`fetch_espn_scoreboard` does, - and nothing is cached then. ``session=None`` uses the fetch service's + and nothing is cached then; nor is an answer whose ``events`` is not a + list (ESPN has sent ``"events": null``), which is returned as it came. ``session=None`` uses the fetch service's pooled session for the ESPN host. """ key = espn_scoreboard_cache_key(sport, league, dates) @@ -969,5 +970,7 @@ def get_espn_scoreboard( # cache read above would have accepted. cache_max_age=None if max_age is None else max(0.0, float(max_age)), ) - store_espn_scoreboard_cache(cache_manager, key, data) + # Every reader of the shared key would inherit a malformed answer. + if isinstance(data, dict) and isinstance(data.get("events"), list): + store_espn_scoreboard_cache(cache_manager, key, data) return data diff --git a/src/common/sports_live_fetch.py b/src/common/sports_live_fetch.py new file mode 100644 index 00000000..b357c6de --- /dev/null +++ b/src/common/sports_live_fetch.py @@ -0,0 +1,200 @@ +"""The live poll's scoreboard request and the odds requests (sports family 9). + +The scoreboards' live fetch and odds code, reconciled in ledmatrix-plugins +(family 9) from seven and three bodies into one each, and copied here under +the existing names. All of it lives on the plugins' ``SportsCore``, so there +is one mixin, ``SportsLiveFetchMixin``: + +- ``_fetch_todays_games()``: today's scoreboard for the live poll, with + yesterday's while it can still hold a live game (``_needs_previous_day``), + as ``{"events": [...]}``; None when ESPN cannot be reached. +- ``_fetch_odds(game)``: queue one game for an odds request; returns at once. +- ``_drain_odds_queue()``: the odds worker that answers the queue. + +THE RULES (decided 2026-10-12) +------------------------------ +Live scores are always fresh. Every poll asks ESPN (``get_espn_scoreboard`` +with ``max_age=1``: only a copy another reader fetched within the last second, +the managers of one display tick, is reused) and leaves the answer under the +shared ``espn_scoreboard_cache_key`` for its dates, so odds-ticker and every +other reader of the same day share it. The plugins' old 30 s cache under +``_scoreboard_current`` is gone. College basketball is asked by +date like every other league; ESPN answered single dates for it on 2026-10-12 +(the old no-dates request cited a 404 that no longer happens). An answer whose +``events`` is not a list counts as no games. + +Odds never block ``update()``. ``_fetch_odds`` puts the game on the manager's +queue and starts one daemon worker if none is running; the worker asks the +odds manager for each queued game in turn and attaches the line to every +dict queued for that id, in place, under ``_games_lock`` (display() reads the +games under it), so a card can draw once without odds and gain them when +ESPN answers. One batch in flight per manager: a game queued while the worker +is busy joins its batch, so a slow endpoint never stacks threads or requests. +The odds manager bounds each request (core's ``BaseOddsManager``: 5 s, then a +60 s hold-off). A live game's line is cached ``live_odds_update_interval`` +(60 s), any other ``odds_update_interval`` (3600 s). Without ``show_odds``, a +game id or an odds manager nothing is asked (the last logged at debug). + +Before the reconcile, six plugins waited on a worker for 1.5 s (2 s for a +live game) per game and hockey and lacrosse asked on the calling thread: with +a 3 s endpoint, ``update()`` of four upcoming games took 6 s and 12 s, and in +the six the line appeared only at the next ``update()`` (an hour later for an +upcoming game). + +A new module rather than more methods on ``sports_fetch``, for the reason +``sports_helpers`` gives: a missing module fails at load, where the version +checks see it; a missing method fails mid-update. + +WHAT A HOST MUST PROVIDE +------------------------ +Derived by walking every ``self.`` the mixin reads; the host-contract +test in ``test/test_sports_live_fetch.py`` fails if a read is added without +being listed here. Every scoreboard ``SportsCore`` supplies all of them. + +- ``session``, ``headers``, ``cache_manager``, ``logger``, ``sport``, + ``league`` -- the scoreboard request. +- ``_needs_previous_day`` -- from ``SportsFetchMixin``. +- ``show_odds``, ``mode_config`` (the two odds intervals), ``sport_key`` (the + worker's thread name), ``_games_lock`` (a lock display() also holds). +- ``odds_manager``, read with ``getattr`` -- ``get_odds(sport=, league=, + event_id=, update_interval_seconds=)``. +- ``_odds_pending`` and ``_odds_worker_busy`` -- the queue and whether a + worker is running; created here on first use, under ``_games_lock``. + +The game dict's ``id`` and ``is_live`` are read; ``odds`` is written. + +BASE ORDER +---------- +No other mixin defines these names, so the position in the bases does not +change which body runs; a method on the plugin's own class still wins. The +mixin has no ``__init__``. +""" + +import threading +from datetime import datetime, timedelta +from typing import Any, Callable, Dict, List, Optional, Tuple + +import pytz +import requests + +from src.common.espn_dates import get_espn_scoreboard + + +class SportsLiveFetchMixin: + """``SportsCore``'s live fetch and odds. See module docstring.""" + + # The host contract, declared for type checking only. + session: Any + headers: Dict[str, str] + cache_manager: Any + logger: Any + sport: str + league: str + sport_key: str + show_odds: bool + mode_config: Dict[str, Any] + _games_lock: Any + _needs_previous_day: Callable[[datetime], bool] + odds_manager: Any + _odds_pending: Dict[Any, Tuple[List[Dict], Any]] + _odds_worker_busy: bool + + def _fetch_todays_games(self) -> Optional[Dict]: + """Today's scoreboard for the live poll, with yesterday's while it can + still hold a live game (``_needs_previous_day``).""" + # ESPN anchors its calendar to Eastern US time. + now = datetime.now(pytz.timezone("America/New_York")) + today = now.strftime("%Y%m%d") + dates = ( + f"{(now - timedelta(days=1)).strftime('%Y%m%d')}-{today}" + if self._needs_previous_day(now) else today + ) + try: + # Asked on every poll and left under core's shared key for this + # scoreboard and date, so odds-ticker and every other reader of + # the same day reuse it. max_age=1: a copy another reader fetched + # within the last second (the managers of one display tick) is the + # same answer; anything older is asked again. + data = get_espn_scoreboard( + self.session, self.sport, self.league, dates, + cache_manager=self.cache_manager, max_age=1, + headers=self.headers, timeout=10, logger=self.logger, + ) + except requests.exceptions.RequestException as e: + self.logger.error( + f"API error fetching today's games for {self.sport} - {self.league}: {e}" + ) + return None + events = data.get("events") if isinstance(data, dict) else None + if not isinstance(events, list): + events = [] # ESPN can answer "events": null + self.logger.info( + f"Fetched {len(events)} games for {dates} for {self.sport} - {self.league}" + ) + return {"events": events} + + def _fetch_odds(self, game: Dict) -> None: + """Queue ``game`` for an odds request; never waits for the answer. + + One daemon worker per manager asks the odds manager for each queued + game in turn and attaches the line to the game dict in place, under + ``_games_lock`` (display() reads the games under it), so a card can + draw once without odds and gain them when ESPN answers. A game queued + while the worker is busy joins its batch, and every dict queued for one + id (update() rebuilds them; the list on screen may keep older ones) + gets the one answer, so a slow endpoint never stacks threads or + requests. The odds manager bounds each request (5 s, then a 60 s + hold-off). + """ + if not self.show_odds or game.get("id") is None: + return + if not getattr(self, "odds_manager", None): + self.logger.debug(f"No odds manager; no odds for game {game['id']}") + return + interval = ( + self.mode_config.get("live_odds_update_interval", 60) + if game.get("is_live", False) + else self.mode_config.get("odds_update_interval", 3600) + ) + with self._games_lock: + pending = getattr(self, "_odds_pending", None) + if pending is None: + pending = self._odds_pending = {} + games, _ = pending.get(game["id"], ([], None)) + if not any(queued is game for queued in games): + games.append(game) + pending[game["id"]] = (games, interval) + if getattr(self, "_odds_worker_busy", False): + return + self._odds_worker_busy = True + try: + threading.Thread(target=self._drain_odds_queue, daemon=True, + name=f"{self.sport_key}-odds").start() + except RuntimeError as e: # no thread to be had; try again next update + with self._games_lock: + self._odds_worker_busy = False + self.logger.debug(f"Could not start the odds worker: {e}") + + def _drain_odds_queue(self) -> None: + """The odds worker: ask for each queued game until none is left.""" + while True: + with self._games_lock: + if not self._odds_pending: + self._odds_worker_busy = False + return + game_id = next(iter(self._odds_pending)) + games, interval = self._odds_pending.pop(game_id) + try: + odds = self.odds_manager.get_odds( + sport=self.sport, + league=self.league, + event_id=game_id, + update_interval_seconds=interval, + ) + except Exception as e: + self.logger.debug(f"Odds fetch failed for game {game_id}: {e}") + continue + if odds: + with self._games_lock: + for game in games: + game["odds"] = odds diff --git a/test/test_espn_scoreboard_cache.py b/test/test_espn_scoreboard_cache.py index 79d92869..05b0bbf2 100644 --- a/test/test_espn_scoreboard_cache.py +++ b/test/test_espn_scoreboard_cache.py @@ -392,6 +392,15 @@ class TestGetEspnScoreboard: "20261004", cache_manager=cache) assert cache.records == {} + @pytest.mark.parametrize("payload", [{"events": None}, {}, {"events": "x"}, []]) + def test_an_answer_without_an_events_list_is_returned_not_cached(self, service, payload): + espn = Espn() + espn.get = lambda url, **kwargs: _response(payload, url, None) + cache = RecordCache() + assert get_espn_scoreboard(espn, "football", "nfl", "20261004", + cache_manager=cache) == payload + assert cache.records == {} + def test_no_cache_manager_just_fetches(self, service): espn = Espn() get_espn_scoreboard(espn, "football", "nfl", "20261004") diff --git a/test/test_sports_live_fetch.py b/test/test_sports_live_fetch.py new file mode 100644 index 00000000..e60531ce --- /dev/null +++ b/test/test_sports_live_fetch.py @@ -0,0 +1,399 @@ +"""src.common.sports_live_fetch: the live poll's request and the odds queue. + +The cases follow ledmatrix-plugins' ``scripts/test_live_fetch_odds.py`` (the +tables the family 9 reconcile was checked against); the expected values are +its columns, which agree across all nine scoreboards. The host is the mixin +over core's ``SportsFetchMixin`` (``_needs_previous_day``), with a session +answering like ESPN did on 2026-10-12 (a single date is 200, a range 400) and +a cache that ages entries on a test clock. Nothing reaches the network. The +odds endpoint is held on an event, so "update() never waits" is shown by the +call returning while the request is still open. +""" + +import ast +import json +import logging +import threading +import time +from datetime import datetime +from pathlib import Path + +import pytest +import requests + +from src.common import espn_dates, sports_live_fetch +from src.common import fetch_service as fs +from src.common.fetch_service import FetchService +from src.common.sports_fetch import SportsFetchMixin +from src.common.sports_live_fetch import SportsLiveFetchMixin + +LOG = logging.getLogger("test_sports_live_fetch") +SITE = "https://site.api.espn.com/apis/site/v2/sports/" + + +def _response(url, answer): + resp = requests.Response() + resp.url = url + resp.status_code = answer if isinstance(answer, int) else 200 + resp._content = answer if isinstance(answer, bytes) else json.dumps( + {} if isinstance(answer, int) else answer).encode() + return resp + + +class Espn(requests.Session): + """Two games a day for a single date; a range is 400, as ESPN answers.""" + + def __init__(self, answer=None): + super().__init__() + self.answer, self.asked = answer, [] + + def get(self, url, params=None, **_kwargs): + dates = (params or {}).get("dates") + self.asked.append((url[len(SITE):], dates)) + if dates and "-" in dates: + return _response(url, 400) + if isinstance(self.answer, type): + raise self.answer("refused by the test") + if self.answer is not None: + return _response(url, self.answer) + return _response(url, {"events": [{"id": dates[4:] + s} for s in "ab"]}) + + +class Cache: + """``get(key, max_age)`` against entries stamped on ``clock``.""" + + def __init__(self, clock): + self.clock, self.store, self.reads = clock, {}, [] + + def get(self, key, max_age=None): + self.reads.append((key, max_age)) + if key not in self.store: + return None + data, stamp = self.store[key] + return data if max_age is None or self.clock() - stamp <= max_age else None + + def set(self, key, data, ttl=None): + self.store[key] = (data, self.clock()) + + +class Clock: + def __init__(self): + self.now = 100_000.0 + + def __call__(self): + return self.now + + +class Frozen(datetime): + at = datetime(2026, 10, 12, 10, 0) + + @classmethod + def now(cls, tz=None): + return tz.localize(cls.at) if tz is not None else cls.at + + +class Host(SportsLiveFetchMixin, SportsFetchMixin): + """A scoreboard SportsCore as far as the family reaches.""" + + sport, league, sport_key = "basketball", "nba", "nba" + + def __init__(self, session=None, cache=None, odds=None, show_odds=True): + self.session, self.headers, self.logger = session or Espn(), {}, LOG + self.cache_manager = cache + self.odds_manager, self.show_odds = odds, show_odds + self.mode_config = {} + self._games_lock = threading.RLock() + + +@pytest.fixture(autouse=True) +def offline(monkeypatch): + clock = Clock() + monkeypatch.setattr(fs, "_service", FetchService({"rate_limits": {}})) + monkeypatch.setattr(espn_dates, "_ranges_rejected_until", 0.0) + monkeypatch.setattr(sports_live_fetch, "datetime", Frozen) + monkeypatch.setattr(time, "time", clock) + Frozen.at = datetime(2026, 10, 12, 10, 0) + return clock + + +# --------------------------------------------------------------------------- +# The live poll's request +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("hour, asked, key, ids", [ + (10, [("20261012",)], "espn_scoreboard_basketball_nba_20261012", ["1012a", "1012b"]), + (2, [("20261011-20261012",), ("20261011",), ("20261012",)], + "espn_scoreboard_basketball_nba_20261011-20261012", + ["1011a", "1011b", "1012a", "1012b"]), +]) +def test_what_the_poll_asks_and_keeps(offline, hour, asked, key, ids): + Frozen.at = datetime(2026, 10, 12, hour, 0) + espn, cache = Espn(), Cache(offline) + data = Host(espn, cache)._fetch_todays_games() + assert [(d,) for _, d in espn.asked] == asked + assert {path for path, _ in espn.asked} == {"basketball/nba/scoreboard"} + assert cache.reads == [(key, 1)] + assert list(cache.store) == [key] + assert [e["id"] for e in data["events"]] == ids + + +def test_college_basketball_is_asked_by_date(offline): + espn, host = Espn(), Host() + host.session, host.league = espn, "mens-college-basketball" + host._fetch_todays_games() + assert espn.asked == [("basketball/mens-college-basketball/scoreboard", "20261012")] + + +@pytest.mark.parametrize("seed_age, later, requests_made", [ + (None, 20, [1, 1]), # every poll asks ESPN; nothing is held 30 s + (None, 0.5, [1, 0]), # a copy from this second is the same answer + (0, None, [0]), # another reader's copy from this second + (2, None, [1]), # ... and one two seconds old is asked again +]) +def test_every_poll_is_fresh(offline, seed_age, later, requests_made): + espn, cache = Espn(), Cache(offline) + if seed_age is not None: + cache.store["espn_scoreboard_basketball_nba_20261012"] = ( + {"events": [{"id": "seed"}]}, offline.now - seed_age) + host, made = Host(espn, cache), [] + for step in ([0] if later is None else [0, later]): + offline.now += step + before = len(espn.asked) + host._fetch_todays_games() + made.append(len(espn.asked) - before) + assert made == requests_made + + +def test_the_old_per_plugin_key_is_not_read(offline): + cache = Cache(offline) + cache.store["nba_scoreboard_current"] = ({"events": [{"id": "old"}]}, offline.now) + data = Host(Espn(), cache)._fetch_todays_games() + assert [e["id"] for e in data["events"]] == ["1012a", "1012b"] + assert "nba_scoreboard_current" not in {k for k, _ in cache.reads} + + +@pytest.mark.parametrize("answer, result, cached", [ + ({}, {"events": []}, False), + ({"events": None}, {"events": []}, False), + ({"events": []}, {"events": []}, True), + (404, None, False), + (500, None, False), + (requests.ConnectionError, None, False), + (requests.Timeout, None, False), + (b"", None, False), +]) +def test_what_a_failed_or_malformed_answer_becomes(offline, answer, result, cached): + cache = Cache(offline) + assert Host(Espn(answer), cache)._fetch_todays_games() == result + assert bool(cache.store) is cached + + +# --------------------------------------------------------------------------- +# Odds +# --------------------------------------------------------------------------- + +class Odds: + """An odds endpoint held open until ``release()``, counting what is in + flight and how many odds workers are alive while it answers.""" + + def __init__(self, answer="odds", held=True): + self.answer, self.calls, self.gate = answer, [], threading.Event() + if not held: + self.gate.set() + self.lock, self.active, self.most_active, self.most_workers = threading.Lock(), 0, 0, 0 + + def get_odds(self, sport, league, event_id, update_interval_seconds=None): + workers = sum(t.name == "nba-odds" and t.is_alive() for t in threading.enumerate()) + with self.lock: + self.calls.append((event_id, update_interval_seconds)) + self.active += 1 + self.most_active = max(self.most_active, self.active) + self.most_workers = max(self.most_workers, workers) + self.gate.wait(10) + with self.lock: + self.active -= 1 + if isinstance(self.answer, type): + raise self.answer("odds endpoint failed") + return {"details": f"{event_id} -3.5"} if self.answer else None + + def release(self): + self.gate.set() + + +def settle(): + for thread in threading.enumerate(): + if thread.name == "nba-odds": + thread.join(10) + + +def game(gid="g1", **flags): + return {"id": gid, **flags} + + +@pytest.mark.parametrize("flags, interval", [({"is_live": True}, 60), ({"is_upcoming": True}, 3600), + ({"is_final": True}, 3600)]) +def test_the_call_never_waits_and_the_line_lands_later(flags, interval): + odds = Odds() + host, g = Host(odds=odds), game(**flags) + host._fetch_odds(g) # returns while the request is held open + assert "odds" not in g + odds.release() + settle() + assert g["odds"] == {"details": "g1 -3.5"} + assert odds.calls == [("g1", interval)] + + +def test_the_intervals_come_from_the_mode_config(): + odds = Odds(held=False) + host = Host(odds=odds) + host.mode_config = {"live_odds_update_interval": 30, "odds_update_interval": 900} + host._fetch_odds(game("l", is_live=True)) + settle() + host._fetch_odds(game("u")) + settle() + assert odds.calls == [("l", 30), ("u", 900)] + + +@pytest.mark.parametrize("answer", [None, RuntimeError]) +def test_no_line_or_a_failed_request_leaves_the_game_bare(caplog, answer): + odds = Odds(answer, held=False) + host, g = Host(odds=odds), game() + with caplog.at_level(logging.DEBUG, logger=LOG.name): + host._fetch_odds(g) + settle() + assert "odds" not in g and host._odds_worker_busy is False + assert [r for r in caplog.records if r.levelno >= logging.WARNING] == [] + + +@pytest.mark.parametrize("host", [ + Host(odds=Odds(held=False), show_odds=False), + Host(odds=None), +], ids=["show_odds off", "no odds manager"]) +def test_nothing_is_asked(caplog, host): + with caplog.at_level(logging.DEBUG, logger=LOG.name): + host._fetch_odds(game()) + assert not [t for t in threading.enumerate() if t.name == "nba-odds"] + assert [r.levelname for r in caplog.records] in ([], ["DEBUG"]) + + +def test_a_game_without_an_id_is_not_asked(): + odds = Odds(held=False) + Host(odds=odds)._fetch_odds({"is_live": True}) + settle() + assert odds.calls == [] + + +def test_one_batch_per_manager_and_every_dict_gets_the_answer(): + odds = Odds() + host = Host(odds=odds) + first, again, other = game("g1"), game("g1"), game("g2") + with host._games_lock: # queued before the worker takes one + for g in (first, again, other): # update() rebuilds the dicts + host._fetch_odds(g) + odds.release() + settle() + assert odds.calls == [("g1", 3600), ("g2", 3600)] + assert odds.most_workers == 1 + assert first["odds"] == again["odds"] == {"details": "g1 -3.5"} + + +def test_a_thread_that_will_not_start_is_tried_again(monkeypatch): + class NoThread: + def __init__(self, *args, **kwargs): + pass + + def start(self): + raise RuntimeError("can't start new thread") + + odds = Odds(held=False) + host = Host(odds=odds) + monkeypatch.setattr(sports_live_fetch.threading, "Thread", NoThread) + host._fetch_odds(game("g1")) + assert host._odds_worker_busy is False + monkeypatch.undo() + host._fetch_odds(game("g2")) + settle() + assert [c[0] for c in odds.calls] == ["g1", "g2"] + + +def test_update_and_display_on_two_threads(): + """Two updaters queueing fresh dicts while a reader walks the games under + the lock, as display() does: no exception, one worker, one request at a + time, and every dict ends with its line.""" + odds = Odds(held=False) + odds.get_odds = _slowed(odds.get_odds) + host, shown, raised = Host(odds=odds), [], [] + + def updater(): + try: + for n in range(30): + g = game(f"g{n % 4}") + with host._games_lock: + shown.append(g) + host._fetch_odds(g) + except Exception as exc: # noqa: BLE001 + raised.append(exc) + + def display(): + try: + for _ in range(300): + with host._games_lock: + [g.get("odds") for g in shown] + except Exception as exc: # noqa: BLE001 + raised.append(exc) + + threads = [threading.Thread(target=updater) for _ in range(2)] + threads.append(threading.Thread(target=display)) + for t in threads: + t.start() + for t in threads: + t.join(30) + settle() + assert raised == [] + assert odds.most_active == 1 and odds.most_workers == 1 + assert all(g.get("odds") for g in shown) + + +def _slowed(get_odds): + def slow(**kwargs): + time.sleep(0.005) + return get_odds(**kwargs) + return slow + + +# --------------------------------------------------------------------------- +# Carrier and host contract +# --------------------------------------------------------------------------- + +MEMBERS = ["_drain_odds_queue", "_fetch_odds", "_fetch_todays_games"] + + +def _self_reads(): + tree = ast.parse(Path(sports_live_fetch.__file__).read_text(encoding="utf-8")) + reads = {node.attr for node in ast.walk(tree) + if isinstance(node, ast.Attribute) and isinstance(node.ctx, ast.Load) + and isinstance(node.value, ast.Name) and node.value.id == "self"} + gets = {node.args[1].value for node in ast.walk(tree) + if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "getattr" + and len(node.args) >= 2 and isinstance(node.args[1], ast.Constant)} + return reads | gets + + +class TestHostContract: + def test_the_mixin_carries_only_the_family(self): + assert sorted(n for n in vars(SportsLiveFetchMixin) if not n.startswith("__")) == MEMBERS + + def test_every_host_read_is_documented(self): + undocumented = sorted(n for n in _self_reads() - set(MEMBERS) + if f"``{n}``" not in sports_live_fetch.__doc__) + assert undocumented == [], f"read but not in the host contract: {undocumented}" + + def test_no_other_shared_mixin_defines_these(self): + from src.common import (sports_display_rules, sports_favorites, sports_helpers, + sports_rankings, sports_rotation, sports_shared) + others = [sports_shared.SportsCoreSharedMixin, sports_shared.SportsRecentSharedMixin, + sports_shared.SportsLiveSharedMixin, sports_helpers.SportsHelpersMixin, + sports_favorites.SportsFavoritesMixin, sports_rotation.SportsRotationMixin, + sports_rankings.SportsRankingsMixin, SportsFetchMixin, + sports_display_rules.SportsGameRulesMixin] + assert [m for m in MEMBERS if any(m in vars(o) for o in others)] == [] diff --git a/test/test_sports_live_fetch_parity.py b/test/test_sports_live_fetch_parity.py new file mode 100644 index 00000000..aab1b1b2 --- /dev/null +++ b/test/test_sports_live_fetch_parity.py @@ -0,0 +1,104 @@ +"""sports_live_fetch still matches every plugin copy. + +``src.common.sports_live_fetch`` was copied from the scoreboards once family 9 +had made each method one body in all nine ``SportsCore`` classes: +``_fetch_todays_games``, ``_fetch_odds`` and ``_drain_odds_queue``. The plugins delete their copies once they floor on the release that +ships this module. Until each has, a copy that changes on its own is a fix one +side has and the other lacks. + +Point LEDMATRIX_PLUGINS at a ledmatrix-plugins checkout and each method is +compared with every plugin copy using ``scripts/sports_drift_report.py``'s own +normalisation (the AST with docstrings and annotations dropped), plus the +decorators. A copy that is gone counts as adopted when the plugin's ``sports.py`` names the module. +Without the variable this skips: core CI has no plugins checkout. +""" + +import ast +import importlib.util +import os +from pathlib import Path + +import pytest + +from src.common import sports_live_fetch + +REPO = Path(__file__).resolve().parents[1] +SPORTS = ("afl", "baseball", "basketball", "football", "hockey", "lacrosse", + "nrl", "soccer", "ufc") +CARRIER, MIXIN = "SportsCore", "SportsLiveFetchMixin" +METHODS = ("_fetch_todays_games", "_fetch_odds", "_drain_odds_queue") + + +def _drift_report(): + """scripts/sports_drift_report.py, loaded by path (scripts/ is no package).""" + spec = importlib.util.spec_from_file_location( + "sports_drift_report", REPO / "scripts" / "sports_drift_report.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +DRIFT = _drift_report() + + +def _plugins_root(): + root = DRIFT.resolve_plugins_dir(os.environ.get("LEDMATRIX_PLUGINS")) + if root is None: + pytest.skip("set LEDMATRIX_PLUGINS to a ledmatrix-plugins checkout to " + "compare this module against the plugin copies") + return root + + +def _class(tree, name): + return next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == name) + + +def _method(cls, name): + return next((n for n in cls.body + if isinstance(n, ast.FunctionDef) and n.name == name), None) + + + +def _fingerprint(node): + return (DRIFT._digest(node, DRIFT._Canonical()), + tuple(ast.unparse(d) for d in node.decorator_list)) + + +def _plugin_tree(root, sport): + source = (root / f"{sport}-scoreboard" / "sports.py").read_text(encoding="utf-8") + return source, ast.parse(source) + + +OURS = _class(ast.parse(Path(sports_live_fetch.__file__).read_text(encoding="utf-8")), MIXIN) + + +@pytest.mark.parametrize("sport, name", [(s, n) for s in SPORTS for n in METHODS]) +def test_every_remaining_plugin_copy_matches(sport, name): + source, tree = _plugin_tree(_plugins_root(), sport) + copy = _method(_class(tree, CARRIER), name) + if copy is None: + assert sports_live_fetch.__name__ in source, ( + f"{sport}: no {name} on {CARRIER} and no {sports_live_fetch.__name__} import") + else: + assert _fingerprint(copy) == _fingerprint(_method(OURS, name)), ( + f"{CARRIER}.{name} in {sport} differs from sports_live_fetch. " + f"Port the change to both, or stop treating it as shared.") + + + +@pytest.mark.parametrize("sport", SPORTS) +def test_no_other_plugin_class_carries_a_copy(sport): + """A copy on another class would shadow the shared one for that class.""" + _, tree = _plugin_tree(_plugins_root(), sport) + strays = [f"{node.name}.{name}" for node in tree.body if isinstance(node, ast.ClassDef) + for name in METHODS if node.name != CARRIER and _method(node, name) is not None] + assert strays == [] + + +def test_the_drift_report_calls_them_identical(): + families = DRIFT.build(_plugins_root(), ("sports.py",)) + rows = {(r["file"], r["family"]): r + for r in (DRIFT.summarise(k, v) for k, v in families.items())} + for name in METHODS: + row = rows.get(("sports.py", name)) + assert row is None or row["worst_class_variants"] == 1, name