rick/LEDMatrix
1
0
Fork 0
mirror of https://github.com/ChuckBuilds/LEDMatrix.git synced 2026-10-13 02:26:37 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
3bfb4a139ef91c9af20d64855dbf600dbfe87e33
LEDMatrix/test/test_espn_scoreboard_cache.py
T
ChuckandClaude Opus 5.5 3bfb4a139e feat(common): sports_live_fetch -- the reconciled live fetch and odds (sports family 9) 
New hardware-free module src/common/sports_live_fetch.py, copied from
ledmatrix-plugins claude/family9-reconcile once the nine scoreboards made
_fetch_todays_games (seven bodies when measured) and _fetch_odds (three) one
body each. Both live on the plugins' SportsCore, so one mixin,
SportsLiveFetchMixin, carries exactly those two plus the odds worker
_drain_odds_queue; no manager gains a method it lacks today.

Owner decisions (2026-10-12):
- Live scores are always fresh: every poll asks ESPN through
  get_espn_scoreboard and leaves the answer under the shared
  espn_scoreboard_cache_key for its dates (max_age=1: only a copy another
  reader fetched within the last second is reused). The plugins' 30 s
  <sport_key>_scoreboard_current cache is gone.
- College basketball is asked by date like every other league (ESPN
  answered single dates for both college leagues on 2026-10-12).
- Odds never block update(): one daemon worker per manager answers a queue
  and attaches each line in place under _games_lock; one batch in flight per
  manager, games queued meanwhile join it, every dict queued for an id gets
  the one answer. show_odds, the two odds intervals and a missing odds
  manager (debug, not ERROR) are respected.
- An answer whose events is not a list is no games.

Fix: get_espn_scoreboard() no longer caches an answer whose events is not a
list ("events": null), which every reader of the shared key inherited; it is
still returned to the caller.

- test/test_sports_live_fetch.py: the plugins' pinned cases (requests,
  dates and keys at 10:00 and 02:00 Eastern, freshness, malformed and failed
  answers, odds that never block and land later, the intervals, one batch per
  manager, a worker that cannot start, update and display on two threads),
  plus the host contract. Mutation-checked (11 mutants, all caught).
- test/test_sports_live_fetch_parity.py: with LEDMATRIX_PLUGINS, compares
  each body with every plugin copy. Passes against claude/family9-reconcile;
  fails against the pre-reconcile tree.
- test/test_espn_scoreboard_cache.py: the null-events regression.
- mypy ratchet, src/common/README.md, CHANGELOG (Unreleased: New modules,
  Fixes), docs/SPORTS_UNIFICATION.md family 9 status and decisions.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 15:45:04 -04:00

508 lines
21 KiB
Python
Raw Blame History

"""One cache key per ESPN scoreboard (fetch service stage 2).
- every core caller names a scoreboard with the same key;
- the keys each consumer used before are read as a fallback;
- a shared entry is never returned older than the reader's own max_age,
whoever wrote it and whatever ttl they stored (checked against a real
CacheManager, whose stored ttl otherwise wins over the reader's);
- two consumers of one scoreboard cost one request;
- the counters see it.
No network: sessions are fakes, and the fetch service is a fresh one per test.
"""
import json
import logging
import os
import threading
import time
from datetime import date, datetime
import pytest
import requests
from requests.structures import CaseInsensitiveDict
from src.common import espn_dates
from src.common import fetch_service as fs
from src.common.espn_dates import (
ESPN_MAX_LIMIT,
espn_scoreboard_cache_key,
espn_scoreboard_cache_key_for_url,
espn_scoreboard_url,
get_espn_scoreboard,
read_espn_scoreboard_cache,
store_espn_scoreboard_cache,
)
from src.common.fetch_service import FetchService, plugin_scope
from src.common.sports_fetch import SportsFetchMixin
NFL_URL = "https://site.api.espn.com/apis/site/v2/sports/football/nfl/scoreboard"
# --- fakes ---------------------------------------------------------------------------
class Clock:
def __init__(self):
self.t = 1000.0
def now(self):
return self.t
def _response(body, url, max_age=None):
response = requests.Response()
response.status_code = 200
response._content = json.dumps(body).encode()
response.headers = CaseInsensitiveDict(
{"Cache-Control": f"max-age={max_age}"} if max_age else {})
response.url = url
response.encoding = "utf-8"
return response
class Espn(requests.Session):
"""A Session answering every scoreboard GET with ``{"events": [<dates>]}``."""
def __init__(self, max_age=None, fail=None):
super().__init__()
self.calls = []
self.max_age = max_age
self.fail = fail
self._lock = threading.Lock()
def get(self, url, **kwargs):
with self._lock:
self.calls.append((url, dict(kwargs.get("params") or {})))
if self.fail:
raise self.fail
dates = (kwargs.get("params") or {}).get("dates", "current")
return _response({"events": [{"id": dates}]}, url, self.max_age)
class RecordCache:
"""The CacheManager surface these helpers use, records and all."""
def __init__(self):
self.records = {}
self.deleted = []
def set(self, key, data, ttl=None):
record = {"timestamp": time.time()}
if ttl is not None:
record["ttl"] = ttl
record["data"] = data
self.records[key] = record
def put(self, key, data, age, ttl=None):
self.set(key, data, ttl)
self.records[key]["timestamp"] -= age
def get_cached_data(self, key, max_age=300, memory_ttl=None):
return self.records.get(key)
def get(self, key, max_age=300, memory_ttl=None):
record = self.records.get(key)
return record["data"] if record else None
def delete(self, key):
self.deleted.append(key)
self.records.pop(key, None)
@pytest.fixture
def service(monkeypatch):
svc = FetchService({"rate_limits": {}})
monkeypatch.setattr(fs, "_service", svc)
return svc
def _totals(svc):
return svc.snapshot()["totals"]
# --- the key -------------------------------------------------------------------------
class TestCanonicalKey:
def test_one_spelling_for_one_scoreboard(self):
key = "espn_scoreboard_football_nfl_20261004"
assert espn_scoreboard_cache_key("football", "nfl", "20261004") == key
assert espn_scoreboard_cache_key(" Football ", "NFL", "20261004") == key
assert espn_scoreboard_cache_key("football", "nfl", date(2026, 10, 4)) == key
assert espn_scoreboard_cache_key("football", "nfl", datetime(2026, 10, 4, 23, 59)) == key
assert espn_scoreboard_cache_key("football", "nfl", ("20261004", date(2026, 10, 4))) == key
assert espn_scoreboard_cache_key_for_url(NFL_URL, "20261004") == key
assert espn_scoreboard_cache_key_for_url(NFL_URL + "?limit=500", "20261004") == key
@pytest.mark.parametrize("dates,suffix", [
(None, "current"),
("", "current"),
("2026", "2026"),
("202610", "202610"),
("20260925-20261016", "20260925-20261016"),
((date(2026, 9, 25), date(2026, 10, 16)), "20260925-20261016"),
])
def test_every_dates_form_espn_takes(self, dates, suffix):
assert espn_scoreboard_cache_key("soccer", "eng.1", dates) == f"espn_scoreboard_soccer_eng.1_{suffix}"
@pytest.mark.parametrize("sport,league,dates", [
("football", "nfl", "2026-10-04"),
("football", "nfl", "tomorrow"),
("football", "nfl", ("20261004",)),
("football", "nfl", True),
("football", "", "20261004"),
("foot ball", "nfl", "20261004"),
("football", "nfl/../x", "20261004"),
])
def test_anything_else_is_refused_not_guessed(self, sport, league, dates):
with pytest.raises(ValueError):
espn_scoreboard_cache_key(sport, league, dates)
def test_a_url_that_is_not_a_scoreboard_has_no_key(self):
assert espn_scoreboard_cache_key_for_url(
"https://site.api.espn.com/apis/site/v2/sports/football/nfl/teams") is None
def test_the_url_it_names(self):
assert espn_scoreboard_url("Football", "college-football") == (
"https://site.api.espn.com/apis/site/v2/sports/football/college-football/scoreboard")
class Host(SportsFetchMixin):
def __init__(self, session, cache):
self.session = session
self.headers = {"User-Agent": "test"}
self.cache_manager = cache
self.logger = logging.getLogger("test.espn_cache")
self._games_lock = threading.RLock()
self.sport = "football"
self.league = "nfl"
class TestEveryCallerUsesTheKey:
"""The core helpers that cache a scoreboard all write the same key."""
DAY = "20261004"
KEY = "espn_scoreboard_football_nfl_20261004"
def test_get_espn_scoreboard(self, service):
cache = RecordCache()
get_espn_scoreboard(Espn(), "football", "nfl", self.DAY, cache_manager=cache)
assert list(cache.records) == [self.KEY]
def test_api_helper(self, service):
from src.common.api_helper import APIHelper
cache = RecordCache()
helper = APIHelper(cache_manager=cache)
helper.set_rate_limit(0)
helper.session = Espn()
helper.fetch_espn_scoreboard("football", "nfl", date=self.DAY)
assert list(cache.records) == [self.KEY]
def test_the_scoreboard_mixin(self, service):
cache = RecordCache()
host = Host(Espn(), cache)
assert host._schedule_cache_key(self.DAY) == self.KEY
host._fetch_season_directly(NFL_URL, self.DAY, None, "today")
assert list(cache.records) == [self.KEY]
def test_the_mixin_without_a_scoreboard_url(self, service):
cache = RecordCache()
Host(Espn(), cache)._fetch_season_directly(
"https://example.test/feed", self.DAY, None, "today")
assert list(cache.records) == [self.KEY]
def test_an_explicit_key_is_still_the_callers(self, service):
cache = RecordCache()
Host(Espn(), cache)._fetch_season_directly(NFL_URL, self.DAY, "mine", "today")
assert list(cache.records) == ["mine"]
# --- reading ---------------------------------------------------------------------------
class TestRead:
KEY = "espn_scoreboard_football_nfl_20261004"
OLD = "scoreboard_data_football_nfl_20261004"
def test_a_fresh_canonical_entry(self, service):
cache = RecordCache()
cache.put(self.KEY, {"events": ["new"]}, age=10)
assert read_espn_scoreboard_cache(cache, self.KEY, 30) == {"events": ["new"]}
assert (_totals(service)["cache_hits"], _totals(service)["legacy_cache_hits"]) == (1, 0)
def test_the_old_key_is_read_after_an_upgrade(self, service):
cache = RecordCache()
cache.put(self.OLD, {"events": ["old"]}, age=10)
assert read_espn_scoreboard_cache(cache, self.KEY, 30, [self.OLD]) == {"events": ["old"]}
assert (_totals(service)["cache_hits"], _totals(service)["legacy_cache_hits"]) == (1, 1)
def test_the_canonical_key_wins_over_an_old_one(self, service):
cache = RecordCache()
cache.put(self.OLD, {"events": ["old"]}, age=1)
cache.put(self.KEY, {"events": ["new"]}, age=20)
assert read_espn_scoreboard_cache(cache, self.KEY, 30, [self.OLD]) == {"events": ["new"]}
def test_a_stale_old_key_is_a_miss_too(self, service):
cache = RecordCache()
cache.put(self.OLD, {"events": ["old"]}, age=31)
assert read_espn_scoreboard_cache(cache, self.KEY, 30, [self.OLD]) is None
assert _totals(service)["cache_hits"] == 0
def test_age_is_judged_against_an_injected_clock(self, service):
cache = RecordCache()
cache.records[self.KEY] = {"timestamp": 5000.0, "data": {"events": []}}
assert read_espn_scoreboard_cache(cache, self.KEY, 30, now=5030.0) == {"events": []}
assert read_espn_scoreboard_cache(cache, self.KEY, 30, now=5030.5) is None
def test_never_older_than_the_readers_ttl_whatever_the_writer_stored(self, service):
# A writer that stored ttl=3600 must not make a 30 s reader take an
# entry a minute old (CacheManager itself would let the ttl win).
cache = RecordCache()
cache.put(self.KEY, {"events": []}, age=60, ttl=3600)
assert read_espn_scoreboard_cache(cache, self.KEY, 30) is None
assert read_espn_scoreboard_cache(cache, self.KEY, 90) == {"events": []}
def test_accept_can_turn_down_an_entry_and_sees_its_age(self, service):
cache = RecordCache()
cache.records[self.KEY] = {"timestamp": 5000.0, "data": {"events": ["live"]}}
seen = []
def live_needs_30s(data, age):
seen.append(age)
return age <= 30
assert read_espn_scoreboard_cache(cache, self.KEY, 300, now=5040.0,
accept=live_needs_30s) is None
assert read_espn_scoreboard_cache(cache, self.KEY, 300, now=5020.0,
accept=live_needs_30s) == {"events": ["live"]}
assert seen == [40.0, 20.0]
assert _totals(service)["cache_hits"] == 1 # the turned-down read is no hit
@pytest.mark.parametrize("max_age", [0, -5])
def test_no_max_age_no_read(self, service, max_age):
cache = RecordCache()
cache.put(self.KEY, {"events": []}, age=0)
assert read_espn_scoreboard_cache(cache, self.KEY, max_age) is None
def test_a_broken_cache_is_a_miss(self, service):
class Broken:
def get_cached_data(self, *a, **k):
raise OSError("disk gone")
assert read_espn_scoreboard_cache(Broken(), self.KEY, 30) is None
assert read_espn_scoreboard_cache(None, self.KEY, 30) is None
def test_a_cache_without_records(self, service):
class Plain:
def get(self, key, max_age=None):
return {"events": [key, max_age]}
assert read_espn_scoreboard_cache(Plain(), self.KEY, 29.5) == {"events": [self.KEY, 30]}
class TestWithARealCacheManager:
"""The same promise against CacheManager, memory and disk tiers."""
KEY = "espn_scoreboard_football_nfl_20261004"
@pytest.fixture
def cm(self, tmp_path):
from src.cache.disk_cache import DiskCache
from src.cache.memory_cache import MemoryCache
from src.cache_manager import CacheManager
cm = CacheManager()
cm._disk_cache_component = DiskCache(cache_dir=str(tmp_path))
cm._memory_cache_component = MemoryCache()
return cm
def _age_on_disk(self, cm, seconds):
path = cm._disk_cache_component.get_cache_path(self.KEY)
with open(path, encoding="utf-8") as fh:
record = json.load(fh)
record["timestamp"] = time.time() - seconds
with open(path, "w", encoding="utf-8") as fh:
json.dump(record, fh)
# Rewriting the file moves its mtime to now, and the disk cache takes a
# record's age from the newer of its timestamp and its mtime (an
# unchanged re-save only touches the file), so age the mtime as well.
os.utime(path, (record["timestamp"], record["timestamp"]))
cm._memory_cache_component.clear()
def test_a_writers_long_ttl_does_not_outlast_the_readers(self, cm, service):
cm.set(self.KEY, {"events": [1]}, ttl=3600)
self._age_on_disk(cm, 60)
assert cm.get(self.KEY, max_age=30) == {"events": [1]} # what a plain read does
assert read_espn_scoreboard_cache(cm, self.KEY, 30) is None
assert read_espn_scoreboard_cache(cm, self.KEY, 120) == {"events": [1]}
def test_a_copy_loaded_into_memory_keeps_its_real_age(self, cm, service):
store_espn_scoreboard_cache(cm, self.KEY, {"events": [1]})
self._age_on_disk(cm, 60)
assert read_espn_scoreboard_cache(cm, self.KEY, 120) == {"events": [1]} # now in memory
assert read_espn_scoreboard_cache(cm, self.KEY, 30) is None
def test_the_canonical_entry_stores_no_ttl(self, cm, service):
store_espn_scoreboard_cache(cm, self.KEY, {"events": []})
path = cm._disk_cache_component.get_cache_path(self.KEY)
with open(path, encoding="utf-8") as fh:
assert "ttl" not in json.load(fh)
# --- fetching through the cache ----------------------------------------------------------------
class TestGetEspnScoreboard:
def test_a_miss_fetches_a_whole_page_and_caches_it(self, service):
cache, espn = RecordCache(), Espn()
data = get_espn_scoreboard(espn, "football", "nfl", "20261004", cache_manager=cache)
assert data == {"events": [{"id": "20261004"}]}
assert espn.calls == [(NFL_URL, {"dates": "20261004", "limit": ESPN_MAX_LIMIT})]
assert "ttl" not in cache.records["espn_scoreboard_football_nfl_20261004"]
def test_a_hit_sends_nothing(self, service):
cache, espn = RecordCache(), Espn()
get_espn_scoreboard(espn, "football", "nfl", "20261004", cache_manager=cache)
get_espn_scoreboard(espn, "football", "nfl", "20261004", cache_manager=cache, max_age=60)
assert len(espn.calls) == 1
assert _totals(service)["cache_hits"] == 1
def test_max_age_zero_always_fetches_and_still_caches(self, service):
cache, espn = RecordCache(), Espn()
get_espn_scoreboard(espn, "football", "nfl", "20261004", cache_manager=cache, max_age=0)
get_espn_scoreboard(espn, "football", "nfl", "20261004", cache_manager=cache, max_age=0)
assert len(espn.calls) == 2
assert "espn_scoreboard_football_nfl_20261004" in cache.records
def test_the_current_scoreboard_sends_no_dates(self, service):
espn = Espn()
get_espn_scoreboard(espn, "football", "nfl", cache_manager=RecordCache())
assert espn.calls == [(NFL_URL, {"limit": ESPN_MAX_LIMIT})]
def test_a_range_is_fetched_in_chunks_once_espn_rejects_it(self, service, monkeypatch):
monkeypatch.setattr(espn_dates, "_ranges_rejected_until", time.monotonic() + 60)
cache, espn = RecordCache(), Espn()
data = get_espn_scoreboard(espn, "football", "nfl", "20261003-20261004", cache_manager=cache)
assert [e["id"] for e in data["events"]] == ["20261003", "20261004"]
assert list(cache.records) == ["espn_scoreboard_football_nfl_20261003-20261004"]
def test_a_failure_raises_and_caches_nothing(self, service):
cache = RecordCache()
with pytest.raises(requests.ConnectionError):
get_espn_scoreboard(Espn(fail=requests.ConnectionError("down")), "football", "nfl",
"20261004", cache_manager=cache)
assert cache.records == {}
@pytest.mark.parametrize("payload", [{"events": None}, {}, {"events": "x"}, []])
def test_an_answer_without_an_events_list_is_returned_not_cached(self, service, payload):
espn = Espn()
espn.get = lambda url, **kwargs: _response(payload, url, None)
cache = RecordCache()
assert get_espn_scoreboard(espn, "football", "nfl", "20261004",
cache_manager=cache) == payload
assert cache.records == {}
def test_no_cache_manager_just_fetches(self, service):
espn = Espn()
get_espn_scoreboard(espn, "football", "nfl", "20261004")
get_espn_scoreboard(espn, "football", "nfl", "20261004")
assert len(espn.calls) == 2
def test_the_response_cache_is_bounded_by_the_callers_ttl(self, monkeypatch):
clock = Clock()
svc = FetchService({"rate_limits": {}}, clock=clock.now)
monkeypatch.setattr(fs, "_service", svc)
espn = Espn(max_age=450)
get_espn_scoreboard(espn, "football", "nfl", "20261004", max_age=0)
clock.t += 20
get_espn_scoreboard(espn, "football", "nfl", "20261004", max_age=10)
assert len(espn.calls) == 2 # 20 s old > the caller's 10
clock.t += 5
get_espn_scoreboard(espn, "football", "nfl", "20261004", max_age=10)
assert len(espn.calls) == 2 # 5 s old, and ESPN said 450
assert svc.snapshot()["totals"]["memo_hits"] == 1
class TestTwoConsumersOneFetch:
"""A scoreboard's live poll and odds-ticker asking for the same day."""
def test_the_second_consumer_is_served_from_the_first_ones_copy(self, service):
cache, espn = RecordCache(), Espn()
with plugin_scope("football-scoreboard"):
live = get_espn_scoreboard(espn, "football", "nfl", "20261004",
cache_manager=cache, max_age=0)
with plugin_scope("odds-ticker"):
ticker = get_espn_scoreboard(
espn, "football", "nfl", "20261004", cache_manager=cache, max_age=300,
legacy_keys=["scoreboard_data_football_nfl_20261004"])
assert ticker == live
assert len(espn.calls) == 1
snap = service.snapshot()
assert snap["plugins"]["odds-ticker"]["cache_hits"] == 1
assert snap["plugins"]["odds-ticker"]["requests"] == 0
assert snap["plugins"]["football-scoreboard"]["requests"] == 1
def test_concurrent_misses_go_to_espn_once(self, service):
gate = threading.Event()
class Slow(Espn):
def get(self, url, **kwargs):
gate.wait(5)
return super().get(url, **kwargs)
espn, results = Slow(), []
threads = [threading.Thread(target=lambda: results.append(get_espn_scoreboard(
espn, "football", "nfl", "20261004", cache_manager=RecordCache(), max_age=60)))
for _ in range(2)]
for thread in threads:
thread.start()
deadline = time.monotonic() + 5
while time.monotonic() < deadline:
with service._lock:
if any(f.waiters for f in service._inflight.values()):
break
time.sleep(0.005)
gate.set()
for thread in threads:
thread.join(5)
assert len(espn.calls) == 1 and results[0] == results[1]
# --- the scoreboards' schedule window ----------------------------------------------------------
class TestCachedSchedule:
KEY = "espn_scoreboard_football_nfl_20260925-20261016"
YESTERDAY = "espn_scoreboard_football_nfl_20260924-20261015"
OLD = "nfl_schedule_window_7_14"
def test_the_canonical_copy(self, service):
cache = RecordCache()
cache.put(self.KEY, {"events": ["new"]}, age=0)
assert Host(Espn(), cache)._cached_schedule(self.KEY, [self.OLD]) == {"events": ["new"]}
assert cache.deleted == []
def test_the_old_key_after_an_upgrade_counted_as_legacy(self, service):
cache = RecordCache()
cache.put(self.OLD, {"events": ["old"]}, age=0)
assert Host(Espn(), cache)._cached_schedule(self.KEY, [self.OLD]) == {"events": ["old"]}
totals = _totals(service)
assert (totals["legacy_cache_hits"], totals["cache_hits"]) == (1, 0)
def test_a_miss_retires_yesterdays_window(self, service):
cache = RecordCache()
cache.put(self.YESTERDAY, {"events": []}, age=0)
assert Host(Espn(), cache)._cached_schedule(self.KEY) is None
assert cache.deleted == [self.YESTERDAY]
assert self.YESTERDAY not in cache.records
def test_a_key_that_is_not_a_window_retires_nothing(self, service):
cache = RecordCache()
Host(Espn(), cache)._cached_schedule("espn_scoreboard_football_nfl_20261004")
Host(Espn(), cache)._cached_schedule(self.OLD)
assert cache.deleted == []
def test_a_cache_without_delete(self, service):
class NoDelete(RecordCache):
delete = None
assert Host(Espn(), NoDelete())._cached_schedule(self.KEY) is None
Reference in New Issue View Git Blame Copy Permalink