mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-10-04 14:25:08 +00:00
ChuckandClaude Opus 5.5
8da7fc01d7
A soccer board with eight leagues logged ~90 NameResolutionErrors for site.api.espn.com within a minute of every start on ledpi, plus "update() timed out". ESPN rejects date ranges, so every league's fortnight-either-side window became 29 day requests, several managers fetched it at once, and each window got its own six chunk threads: ~450 requests per start with ~45 in flight, each request past a session's pool a new connection and a new DNS lookup. - fetch_espn_date_chunks() asks for a partial edge month whole once the window covers ESPN_MONTH_COVER_MIN_DAYS (7) of its days, and trims the answer to the window by each event's US Eastern start date, the day ESPN's dates=YYYYMMDD means (417/417 live soccer events matched). A 29-day window over two months is 2 requests, not 29. A live poll's 1-2 days stay day requests. A capped trimmed month re-asks only the window's days; an undatable event is kept. New espn_request_chunks(). - Chunk requests share one process-wide cap of ESPN_CHUNK_WORKERS in flight instead of six per window. - A new process starts as if a range had just been rejected, so it no longer spends a doomed 400 per window at every start. Live ESPN, soccer-scoreboard 2.39.2, alternating runs: ~450 -> 46 requests per start, peak ~45 -> 13 in flight, ~75 -> 30 DNS lookups. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
660 lines
26 KiB
Python
660 lines
26 KiB
Python
"""src.common.espn_dates: the ESPN site-API workarounds.
|
|
|
|
Two real upstream behaviours are pinned here, both observed on 2026-09-15 and
|
|
re-verified with desktop curl (see the module docstring):
|
|
|
|
* ``dates=YYYYMMDD-YYYYMMDD`` answers 400 for every sport, so a range has to be
|
|
re-asked in months and days.
|
|
* ``limit`` over 500 truncates instead of erroring -- college-football returned
|
|
25 of 68 games for a single Saturday at ``limit=1000``.
|
|
|
|
Nothing here touches the network. The fake session records what a caller would
|
|
have sent, which is the part that regressed.
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
from datetime import date, timedelta
|
|
|
|
import pytest
|
|
|
|
import src.common.espn_dates as espn_dates
|
|
from src.common.espn_dates import (
|
|
ESPN_MAX_LIMIT,
|
|
RANGE_RETRY_SECONDS,
|
|
clamp_espn_limit,
|
|
espn_date_chunks,
|
|
fetch_espn_date_chunks,
|
|
fetch_espn_scoreboard,
|
|
merge_scoreboard_payloads,
|
|
parse_espn_date_range,
|
|
)
|
|
|
|
URL = "https://site.api.espn.com/apis/site/v2/sports/football/nfl/scoreboard"
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def forget_rejected_ranges(monkeypatch):
|
|
"""The rejected-range memo is process-wide; no test may inherit it."""
|
|
monkeypatch.setattr(espn_dates, "_ranges_rejected_until", 0.0)
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(self, status_code=200, payload=None):
|
|
self.status_code = status_code
|
|
self._payload = payload if payload is not None else {"events": []}
|
|
|
|
def json(self):
|
|
return self._payload
|
|
|
|
def raise_for_status(self):
|
|
if self.status_code >= 400:
|
|
raise RuntimeError(str(self.status_code) + " Client Error: Bad Request")
|
|
|
|
|
|
class FakeSession:
|
|
"""Answers 400 to day ranges, like ESPN does, and records every call."""
|
|
|
|
def __init__(self, events_by_chunk=None, fail_chunks=()):
|
|
self.calls = []
|
|
self.events_by_chunk = events_by_chunk or {}
|
|
self.fail_chunks = set(fail_chunks)
|
|
|
|
def get(self, url, params=None, headers=None, timeout=None):
|
|
params = params or {}
|
|
dates = str(params.get("dates", ""))
|
|
self.calls.append(params)
|
|
if parse_espn_date_range(dates) is not None:
|
|
return FakeResponse(400)
|
|
if dates in self.fail_chunks:
|
|
return FakeResponse(500)
|
|
return FakeResponse(200, {"events": self.events_by_chunk.get(dates, [])})
|
|
|
|
|
|
def days_covered_by(chunks):
|
|
"""Expand chunks back into the days they stand for, in order."""
|
|
covered = []
|
|
for chunk in chunks:
|
|
if len(chunk) == 6:
|
|
day = date(int(chunk[:4]), int(chunk[4:]), 1)
|
|
month = day.month
|
|
while day.month == month:
|
|
covered.append(day)
|
|
day += timedelta(days=1)
|
|
else:
|
|
covered.append(date(int(chunk[:4]), int(chunk[4:6]), int(chunk[6:])))
|
|
return covered
|
|
|
|
|
|
class TestClampLimit:
|
|
"""limit over 500 silently truncates upstream, so it must never be sent."""
|
|
|
|
def test_the_limit_every_caller_used_is_pulled_back(self):
|
|
assert clamp_espn_limit({"limit": 1000})["limit"] == ESPN_MAX_LIMIT
|
|
|
|
def test_a_safe_limit_is_left_alone(self):
|
|
assert clamp_espn_limit({"limit": 100})["limit"] == 100
|
|
|
|
def test_the_boundary_value_is_kept(self):
|
|
assert clamp_espn_limit({"limit": 500})["limit"] == 500
|
|
|
|
@pytest.mark.parametrize("params", [{}, {"limit": None}, {"limit": "many"}])
|
|
def test_absent_or_unparseable_limits_pass_through(self, params):
|
|
assert clamp_espn_limit(params) == params
|
|
|
|
def test_the_callers_dict_is_not_mutated(self):
|
|
original = {"limit": 1000}
|
|
clamp_espn_limit(original)
|
|
assert original == {"limit": 1000}
|
|
|
|
|
|
class TestParseRange:
|
|
def test_a_day_range_parses(self):
|
|
assert parse_espn_date_range("20260801-20270301") == (
|
|
date(2026, 8, 1),
|
|
date(2027, 3, 1),
|
|
)
|
|
|
|
@pytest.mark.parametrize(
|
|
"value",
|
|
[
|
|
"20260914", # single day still works upstream
|
|
"202609", # month still works upstream
|
|
"2026", # season year still works upstream
|
|
"20260801-",
|
|
"not-a-date",
|
|
"20270301-20260801", # backwards
|
|
"2026080-20270301", # short half
|
|
None,
|
|
1234,
|
|
],
|
|
)
|
|
def test_everything_that_is_not_a_day_range_is_left_alone(self, value):
|
|
assert parse_espn_date_range(value) is None
|
|
|
|
|
|
class TestChunks:
|
|
"""Chunks must tile the window exactly -- never reaching outside it."""
|
|
|
|
def test_a_full_season_collapses_to_months_plus_one_day(self):
|
|
assert espn_date_chunks(date(2026, 8, 1), date(2027, 3, 1)) == [
|
|
"202608",
|
|
"202609",
|
|
"202610",
|
|
"202611",
|
|
"202612",
|
|
"202701",
|
|
"202702",
|
|
"20270301",
|
|
]
|
|
|
|
def test_a_two_day_window_stays_two_days(self):
|
|
assert espn_date_chunks(date(2026, 9, 14), date(2026, 9, 15)) == [
|
|
"20260914",
|
|
"20260915",
|
|
]
|
|
|
|
def test_an_exact_calendar_month_is_one_request(self):
|
|
assert espn_date_chunks(date(2026, 9, 1), date(2026, 9, 30)) == ["202609"]
|
|
|
|
def test_partial_edges_are_spelled_out_day_by_day(self):
|
|
assert espn_date_chunks(date(2026, 8, 30), date(2026, 10, 2)) == [
|
|
"20260830",
|
|
"20260831",
|
|
"202609",
|
|
"20261001",
|
|
"20261002",
|
|
]
|
|
|
|
def test_a_single_day_window_is_one_day(self):
|
|
assert espn_date_chunks(date(2026, 9, 14), date(2026, 9, 14)) == ["20260914"]
|
|
|
|
def test_february_in_a_leap_year_is_still_one_month(self):
|
|
assert espn_date_chunks(date(2028, 2, 1), date(2028, 2, 29)) == ["202802"]
|
|
|
|
def test_the_29th_of_a_leap_february_is_not_swallowed(self):
|
|
# A month chunk may only be used when it ends inside the window.
|
|
assert espn_date_chunks(date(2028, 2, 1), date(2028, 2, 28)) == [
|
|
"202802{:02d}".format(day) for day in range(1, 29)
|
|
]
|
|
|
|
def test_a_year_boundary_is_crossed_cleanly(self):
|
|
assert espn_date_chunks(date(2026, 12, 31), date(2027, 1, 31)) == [
|
|
"20261231",
|
|
"202701",
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
"start,end",
|
|
[
|
|
(date(2026, 8, 1), date(2027, 3, 1)),
|
|
(date(2026, 8, 30), date(2026, 10, 2)),
|
|
(date(2025, 9, 1), date(2026, 8, 1)),
|
|
(date(2026, 9, 14), date(2026, 9, 15)),
|
|
],
|
|
)
|
|
def test_chunks_cover_every_day_exactly_once(self, start, end):
|
|
expected = []
|
|
day = start
|
|
while day <= end:
|
|
expected.append(day)
|
|
day += timedelta(days=1)
|
|
assert days_covered_by(espn_date_chunks(start, end)) == expected
|
|
|
|
|
|
class TestMerge:
|
|
def test_events_are_deduplicated_by_id(self):
|
|
merged = merge_scoreboard_payloads(
|
|
[
|
|
{"events": [{"id": "1"}, {"id": "2"}]},
|
|
{"events": [{"id": "2"}, {"id": "3"}]},
|
|
]
|
|
)
|
|
assert [e["id"] for e in merged["events"]] == ["1", "2", "3"]
|
|
|
|
def test_non_event_keys_come_from_the_first_payload_that_has_them(self):
|
|
merged = merge_scoreboard_payloads(
|
|
[
|
|
{"events": [], "leagues": ["first"]},
|
|
{"events": [], "leagues": ["second"], "season": 2026},
|
|
]
|
|
)
|
|
assert merged["leagues"] == ["first"]
|
|
assert merged["season"] == 2026
|
|
|
|
def test_an_empty_merge_still_has_an_events_list(self):
|
|
assert merge_scoreboard_payloads([]) == {"events": []}
|
|
|
|
|
|
class TestFetch:
|
|
def test_a_working_request_is_not_chunked(self):
|
|
session = FakeSession({"20260913": [{"id": "1"}]})
|
|
data = fetch_espn_scoreboard(session, URL, params={"dates": "20260913"})
|
|
assert data["events"] == [{"id": "1"}]
|
|
assert len(session.calls) == 1
|
|
|
|
def test_limit_is_clamped_even_on_the_happy_path(self):
|
|
session = FakeSession()
|
|
fetch_espn_scoreboard(session, URL, params={"dates": "20260913", "limit": 1000})
|
|
assert session.calls[0]["limit"] == ESPN_MAX_LIMIT
|
|
|
|
def test_a_rejected_range_is_refetched_in_chunks(self):
|
|
session = FakeSession(
|
|
{"202609": [{"id": "a"}, {"id": "b"}], "20261001": [{"id": "c"}]}
|
|
)
|
|
data = fetch_espn_scoreboard(
|
|
session, URL, params={"dates": "20260901-20261001", "limit": 1000}
|
|
)
|
|
assert [e["id"] for e in data["events"]] == ["a", "b", "c"]
|
|
sent = [call["dates"] for call in session.calls]
|
|
# Chunks race, so only the rejected range is pinned to a position --
|
|
# the merged event order above is what has to stay deterministic.
|
|
assert sent[0] == "20260901-20261001"
|
|
assert sorted(sent[1:]) == ["202609", "20261001"]
|
|
|
|
def test_chunk_requests_keep_the_clamped_limit(self):
|
|
session = FakeSession({"202609": []})
|
|
fetch_espn_scoreboard(
|
|
session, URL, params={"dates": "20260901-20260930", "limit": 1000}
|
|
)
|
|
assert all(call["limit"] == ESPN_MAX_LIMIT for call in session.calls)
|
|
|
|
def test_other_params_survive_chunking(self):
|
|
session = FakeSession({"202609": []})
|
|
fetch_espn_scoreboard(
|
|
session, URL, params={"dates": "20260901-20260930", "groups": "80"}
|
|
)
|
|
assert session.calls[-1]["groups"] == "80"
|
|
|
|
def test_one_bad_chunk_does_not_sink_the_season(self):
|
|
session = FakeSession(
|
|
{"202609": [{"id": "a"}], "20261001": [{"id": "c"}]},
|
|
fail_chunks={"20261001"},
|
|
)
|
|
data = fetch_espn_scoreboard(session, URL, params={"dates": "20260901-20261001"})
|
|
assert [e["id"] for e in data["events"]] == ["a"]
|
|
|
|
def test_a_total_failure_raises_rather_than_looking_like_no_games(self):
|
|
session = FakeSession({}, fail_chunks={"202609"})
|
|
with pytest.raises(RuntimeError):
|
|
fetch_espn_scoreboard(session, URL, params={"dates": "20260901-20260930"})
|
|
|
|
def test_a_400_on_a_non_range_request_is_still_an_error(self):
|
|
class AlwaysBad(FakeSession):
|
|
def get(self, url, params=None, headers=None, timeout=None):
|
|
self.calls.append(params or {})
|
|
return FakeResponse(400)
|
|
|
|
session = AlwaysBad()
|
|
with pytest.raises(RuntimeError):
|
|
fetch_espn_scoreboard(session, URL, params={"dates": "20260913"})
|
|
assert len(session.calls) == 1
|
|
|
|
|
|
class TestMonthCap:
|
|
"""A month holding more than 500 events comes back cut at exactly 500.
|
|
|
|
College baseball's March 2026 does this. Nothing in the response says more
|
|
exist, so a full month chunk has to be re-asked day by day.
|
|
"""
|
|
|
|
def test_a_full_month_is_re_asked_day_by_day(self):
|
|
full = [{"id": "m%d" % i} for i in range(ESPN_MAX_LIMIT)]
|
|
by_chunk = {"202603": full}
|
|
by_chunk.update({"202603%02d" % day: [{"id": "d%d" % day}] for day in range(1, 32)})
|
|
session = FakeSession(by_chunk)
|
|
|
|
data = fetch_espn_date_chunks(session, URL, params={"dates": "20260301-20260331"})
|
|
|
|
sent = [call["dates"] for call in session.calls]
|
|
# The month has to be asked before its days can be known to be needed;
|
|
# the days themselves race, so compare them as a set.
|
|
assert sent[0] == "202603"
|
|
assert sorted(sent[1:]) == ["202603%02d" % day for day in range(1, 32)]
|
|
# The truncated month payload is dropped, not merged with the days.
|
|
assert [event["id"] for event in data["events"]] == [
|
|
"d%d" % day for day in range(1, 32)
|
|
]
|
|
|
|
def test_a_month_under_the_cap_is_trusted(self):
|
|
session = FakeSession({"202609": [{"id": "a"}] * 10})
|
|
fetch_espn_date_chunks(session, URL, params={"dates": "20260901-20260930"})
|
|
assert [call["dates"] for call in session.calls] == ["202609"]
|
|
|
|
def test_chunks_ask_for_the_cap_even_when_the_caller_sent_no_limit(self):
|
|
# ESPN's default page is 100 for NFL and 300 for college football --
|
|
# smaller than a busy month.
|
|
session = FakeSession({"202609": []})
|
|
fetch_espn_date_chunks(session, URL, params={"dates": "20260901-20260930"})
|
|
assert session.calls[0]["limit"] == ESPN_MAX_LIMIT
|
|
|
|
def test_a_non_range_is_not_chunked(self):
|
|
session = FakeSession()
|
|
assert fetch_espn_date_chunks(session, URL, params={"dates": "202609"}) is None
|
|
assert session.calls == []
|
|
|
|
|
|
class TestRejectedRangeMemo:
|
|
"""Live boards ask every 30s; a known-rejected range must not be re-sent."""
|
|
|
|
def test_after_one_rejection_the_next_range_skips_straight_to_chunks(self):
|
|
session = FakeSession({"20260914": [{"id": "a"}], "20260915": []})
|
|
fetch_espn_scoreboard(session, URL, params={"dates": "20260914-20260915"})
|
|
session.calls.clear()
|
|
|
|
data = fetch_espn_scoreboard(session, URL, params={"dates": "20260914-20260915"})
|
|
|
|
assert [call["dates"] for call in session.calls] == ["20260914", "20260915"]
|
|
assert [event["id"] for event in data["events"]] == ["a"]
|
|
|
|
def test_the_range_is_tried_again_once_the_memo_expires(self, monkeypatch):
|
|
clock = [1000.0]
|
|
monkeypatch.setattr(espn_dates.time, "monotonic", lambda: clock[0])
|
|
session = FakeSession()
|
|
fetch_espn_scoreboard(session, URL, params={"dates": "20260914-20260915"})
|
|
session.calls.clear()
|
|
|
|
clock[0] += RANGE_RETRY_SECONDS + 1
|
|
fetch_espn_scoreboard(session, URL, params={"dates": "20260914-20260915"})
|
|
|
|
assert session.calls[0]["dates"] == "20260914-20260915"
|
|
|
|
def test_a_400_on_a_single_day_does_not_mark_ranges_rejected(self):
|
|
class AlwaysBad(FakeSession):
|
|
def get(self, url, params=None, headers=None, timeout=None):
|
|
self.calls.append(params or {})
|
|
return FakeResponse(400)
|
|
|
|
with pytest.raises(RuntimeError):
|
|
fetch_espn_scoreboard(AlwaysBad(), URL, params={"dates": "20260913"})
|
|
assert not espn_dates._ranges_known_rejected()
|
|
|
|
def test_when_every_chunk_fails_the_range_itself_supplies_the_error(self):
|
|
session = FakeSession(fail_chunks={"20260914", "20260915"})
|
|
fetch_espn_scoreboard(session, URL, params={"dates": "20260801-20260801"})
|
|
session.calls.clear()
|
|
|
|
with pytest.raises(RuntimeError):
|
|
fetch_espn_scoreboard(session, URL, params={"dates": "20260914-20260915"})
|
|
assert session.calls[-1]["dates"] == "20260914-20260915"
|
|
|
|
|
|
class TestConcurrency:
|
|
"""Chunks go out in parallel, which must not change what comes back.
|
|
|
|
A cold college-baseball season is ~130 chunks once February through May
|
|
are re-asked day by day. Sequentially that outran the 20s plugin update()
|
|
timeout on a Pi, so the requests now overlap -- but the merged payload has
|
|
to stay exactly what the sequential version produced.
|
|
"""
|
|
|
|
def test_events_keep_chunk_order_however_the_requests_race(self):
|
|
# Answer the later chunks fastest, so completion order is the reverse
|
|
# of chunk order and a naive gather would interleave them wrongly.
|
|
class RacingSession(FakeSession):
|
|
def get(self, url, params=None, headers=None, timeout=None):
|
|
dates = str((params or {}).get("dates", ""))
|
|
if len(dates) == 6:
|
|
time.sleep(0.02 / (int(dates[4:]) or 1))
|
|
return super().get(url, params=params, headers=headers, timeout=timeout)
|
|
|
|
session = RacingSession(
|
|
{
|
|
"202609": [{"id": "sep"}],
|
|
"202610": [{"id": "oct"}],
|
|
"202611": [{"id": "nov"}],
|
|
}
|
|
)
|
|
data = fetch_espn_date_chunks(
|
|
session, URL, params={"dates": "20260901-20261130"}
|
|
)
|
|
assert [event["id"] for event in data["events"]] == ["sep", "oct", "nov"]
|
|
|
|
def test_a_capped_month_splices_its_days_in_place(self):
|
|
# October is capped and expands to 31 days; September and November
|
|
# must still bracket those days in the merged result.
|
|
full = [{"id": "cap%d" % i} for i in range(ESPN_MAX_LIMIT)]
|
|
by_chunk = {
|
|
"202609": [{"id": "sep"}],
|
|
"202610": full,
|
|
"202611": [{"id": "nov"}],
|
|
}
|
|
by_chunk.update(
|
|
{"202610%02d" % day: [{"id": "oct%02d" % day}] for day in range(1, 32)}
|
|
)
|
|
session = FakeSession(by_chunk)
|
|
|
|
data = fetch_espn_date_chunks(
|
|
session, URL, params={"dates": "20260901-20261130"}
|
|
)
|
|
|
|
expected = ["sep"] + ["oct%02d" % day for day in range(1, 32)] + ["nov"]
|
|
assert [event["id"] for event in data["events"]] == expected
|
|
|
|
def test_two_capped_months_expand_without_crossing_over(self):
|
|
full = [{"id": "cap%d" % i} for i in range(ESPN_MAX_LIMIT)]
|
|
by_chunk = {"202609": full, "202610": full}
|
|
by_chunk.update(
|
|
{"202609%02d" % day: [{"id": "s%02d" % day}] for day in range(1, 31)}
|
|
)
|
|
by_chunk.update(
|
|
{"202610%02d" % day: [{"id": "o%02d" % day}] for day in range(1, 32)}
|
|
)
|
|
session = FakeSession(by_chunk)
|
|
|
|
data = fetch_espn_date_chunks(
|
|
session, URL, params={"dates": "20260901-20261031"}
|
|
)
|
|
|
|
expected = ["s%02d" % day for day in range(1, 31)] + [
|
|
"o%02d" % day for day in range(1, 32)
|
|
]
|
|
assert [event["id"] for event in data["events"]] == expected
|
|
|
|
def test_a_failed_day_inside_a_capped_month_only_costs_that_day(self):
|
|
full = [{"id": "cap%d" % i} for i in range(ESPN_MAX_LIMIT)]
|
|
by_chunk = {"202610": full}
|
|
by_chunk.update(
|
|
{"202610%02d" % day: [{"id": "o%02d" % day}] for day in range(1, 32)}
|
|
)
|
|
session = FakeSession(by_chunk, fail_chunks={"20261015"})
|
|
|
|
data = fetch_espn_date_chunks(
|
|
session, URL, params={"dates": "20261001-20261031"}
|
|
)
|
|
|
|
expected = ["o%02d" % day for day in range(1, 32) if day != 15]
|
|
assert [event["id"] for event in data["events"]] == expected
|
|
|
|
def test_no_more_than_the_worker_cap_are_in_flight_at_once(self):
|
|
live = {"now": 0, "peak": 0}
|
|
guard = threading.Lock()
|
|
|
|
class CountingSession(FakeSession):
|
|
def get(self, url, params=None, headers=None, timeout=None):
|
|
with guard:
|
|
live["now"] += 1
|
|
live["peak"] = max(live["peak"], live["now"])
|
|
try:
|
|
time.sleep(0.01)
|
|
return super().get(
|
|
url, params=params, headers=headers, timeout=timeout
|
|
)
|
|
finally:
|
|
with guard:
|
|
live["now"] -= 1
|
|
|
|
session = CountingSession()
|
|
fetch_espn_date_chunks(session, URL, params={"dates": "20260101-20261231"})
|
|
|
|
assert live["peak"] <= espn_dates.ESPN_CHUNK_WORKERS
|
|
assert live["peak"] > 1, "chunks should actually overlap"
|
|
|
|
|
|
class TestEdgeMonths:
|
|
"""A window's partial edge months are asked whole and trimmed.
|
|
|
|
The default scoreboard window -- a fortnight either side of today -- spans
|
|
two partial months, so it used to cost 29 day requests per league. ESPN's
|
|
``dates=YYYYMMDD`` means a US Eastern day (verified against the live API
|
|
on 2026-10-03, 417 of 417 soccer events), so a month answer trimmed to
|
|
the window's Eastern days is what the day requests returned.
|
|
"""
|
|
|
|
def test_a_fortnight_either_side_is_two_requests(self):
|
|
planned = espn_dates.espn_request_chunks(date(2026, 9, 20), date(2026, 10, 18))
|
|
assert planned == [
|
|
("202609", (date(2026, 9, 20), date(2026, 9, 30))),
|
|
("202610", (date(2026, 10, 1), date(2026, 10, 18))),
|
|
]
|
|
|
|
def test_a_live_polls_two_days_stay_two_days(self):
|
|
planned = espn_dates.espn_request_chunks(date(2026, 10, 2), date(2026, 10, 3))
|
|
assert planned == [("20261002", None), ("20261003", None)]
|
|
|
|
def test_the_threshold_is_inclusive(self):
|
|
n = espn_dates.ESPN_MONTH_COVER_MIN_DAYS
|
|
short = espn_dates.espn_request_chunks(date(2026, 10, 1), date(2026, 10, n - 1))
|
|
assert [chunk for chunk, _ in short] == [
|
|
"202610%02d" % day for day in range(1, n)]
|
|
enough = espn_dates.espn_request_chunks(date(2026, 10, 1), date(2026, 10, n))
|
|
assert enough == [("202610", (date(2026, 10, 1), date(2026, 10, n)))]
|
|
|
|
def test_whole_months_and_short_edges_are_unchanged(self):
|
|
planned = espn_dates.espn_request_chunks(date(2026, 8, 30), date(2026, 10, 2))
|
|
assert planned == [
|
|
("20260830", None), ("20260831", None), ("202609", None),
|
|
("20261001", None), ("20261002", None),
|
|
]
|
|
|
|
def test_without_time_zone_data_edges_stay_days(self, monkeypatch):
|
|
monkeypatch.setattr(espn_dates, "_EASTERN", None)
|
|
planned = espn_dates.espn_request_chunks(date(2026, 9, 20), date(2026, 10, 18))
|
|
assert len(planned) == 29
|
|
assert all(trim is None for _, trim in planned)
|
|
|
|
@pytest.mark.parametrize("start,end", [
|
|
(date(2026, 9, 20), date(2026, 10, 18)),
|
|
(date(2026, 1, 25), date(2026, 3, 3)),
|
|
(date(2026, 12, 20), date(2027, 1, 9)),
|
|
(date(2026, 10, 5), date(2026, 10, 9)),
|
|
])
|
|
def test_the_planned_requests_still_cover_every_day_exactly_once(self, start, end):
|
|
covered = []
|
|
for chunk, trim in espn_dates.espn_request_chunks(start, end):
|
|
if trim is None:
|
|
covered.extend(days_covered_by([chunk]))
|
|
else:
|
|
assert chunk == trim[0].strftime("%Y%m") == trim[1].strftime("%Y%m")
|
|
covered.extend(trim[0] + timedelta(days=offset)
|
|
for offset in range((trim[1] - trim[0]).days + 1))
|
|
expected = [start + timedelta(days=offset) for offset in range((end - start).days + 1)]
|
|
assert covered == expected
|
|
|
|
def test_a_trimmed_month_keeps_only_the_windows_eastern_days(self):
|
|
september = [
|
|
# 03:30Z on the 20th is still the 19th in New York: outside.
|
|
{"id": "before", "date": "2026-09-20T03:30Z"},
|
|
{"id": "first", "date": "2026-09-20T14:00Z"},
|
|
{"id": "late", "date": "2026-09-30T23:30Z"},
|
|
]
|
|
october = [
|
|
{"id": "oct1", "date": "2026-10-01T19:00Z"},
|
|
# 03:30Z on the 19th is the evening of the 18th in New York: inside.
|
|
{"id": "last", "date": "2026-10-19T03:30Z"},
|
|
{"id": "after", "date": "2026-10-19T14:00Z"},
|
|
{"id": "undated"},
|
|
]
|
|
session = FakeSession({"202609": september, "202610": october})
|
|
|
|
data = fetch_espn_date_chunks(session, URL, params={"dates": "20260920-20261018"})
|
|
|
|
assert sorted(call["dates"] for call in session.calls) == ["202609", "202610"]
|
|
# An event with no readable date is kept, never dropped on a guess.
|
|
assert [e["id"] for e in data["events"]] == ["first", "late", "oct1", "last", "undated"]
|
|
|
|
def test_eastern_standard_time_is_honoured_after_the_clocks_change(self):
|
|
# 2026-11-01 ends daylight saving: Eastern is UTC-5 from then on.
|
|
november = [
|
|
{"id": "out", "date": "2026-11-15T04:30Z"}, # Nov 14, 23:30 EST
|
|
{"id": "in", "date": "2026-11-15T05:30Z"}, # Nov 15, 00:30 EST
|
|
]
|
|
session = FakeSession({"202611": november})
|
|
data = fetch_espn_date_chunks(session, URL, params={"dates": "20261115-20261121"})
|
|
assert [e["id"] for e in data["events"]] == ["in"]
|
|
|
|
def test_a_capped_edge_month_re_asks_only_the_windows_days(self):
|
|
full = [{"id": "cap%d" % i, "date": "2026-10-05T18:00Z"} for i in range(ESPN_MAX_LIMIT)]
|
|
by_chunk = {"202610": full}
|
|
by_chunk.update({"202610%02d" % day: [{"id": "o%02d" % day}] for day in range(1, 32)})
|
|
session = FakeSession(by_chunk)
|
|
|
|
data = fetch_espn_date_chunks(session, URL, params={"dates": "20261001-20261010"})
|
|
|
|
sent = [call["dates"] for call in session.calls]
|
|
assert sent[0] == "202610"
|
|
assert sorted(sent[1:]) == ["202610%02d" % day for day in range(1, 11)]
|
|
assert [e["id"] for e in data["events"]] == ["o%02d" % day for day in range(1, 11)]
|
|
|
|
|
|
class TestProcessWideChunkCap:
|
|
"""The chunk cap holds across windows, not per window.
|
|
|
|
A soccer board starting eight leagues fetches sixteen windows at once.
|
|
With a pool of ``ESPN_CHUNK_WORKERS`` each, ~40 requests were in flight
|
|
and every one past a session's pool opened a connection -- and a DNS
|
|
lookup. On ledpi that was ~90 NameResolutionErrors per start.
|
|
"""
|
|
|
|
def test_concurrent_windows_share_one_budget(self):
|
|
live = {"now": 0, "peak": 0}
|
|
guard = threading.Lock()
|
|
|
|
class CountingSession(FakeSession):
|
|
def get(self, url, params=None, headers=None, timeout=None):
|
|
with guard:
|
|
live["now"] += 1
|
|
live["peak"] = max(live["peak"], live["now"])
|
|
try:
|
|
time.sleep(0.01)
|
|
return super().get(url, params=params, headers=headers, timeout=timeout)
|
|
finally:
|
|
with guard:
|
|
live["now"] -= 1
|
|
|
|
sessions = [CountingSession() for _ in range(6)]
|
|
# Six leagues, so the fetch service cannot merge them into one, on a
|
|
# host with no token bucket: earlier tests may have spent ESPN's
|
|
# burst, and a bucket paced at 20/s would serialise these by itself.
|
|
threads = [
|
|
threading.Thread(target=fetch_espn_date_chunks,
|
|
args=(session, "https://scores.example.test/league%d" % index),
|
|
kwargs={"params": {"dates": "20260101-20261231"}})
|
|
for index, session in enumerate(sessions)
|
|
]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join(timeout=30)
|
|
|
|
assert all(len(session.calls) == 12 for session in sessions)
|
|
assert live["peak"] <= espn_dates.ESPN_CHUNK_WORKERS
|
|
assert live["peak"] > 1, "chunks should still overlap"
|
|
|
|
|
|
def test_a_fresh_process_skips_the_doomed_range_request():
|
|
"""Every start used to spend one 400 per window learning that ranges are
|
|
still rejected -- eleven at once from a soccer board. A new process now
|
|
starts inside the retry period instead."""
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
out = subprocess.run(
|
|
[sys.executable, "-c",
|
|
"import src.common.espn_dates as e; print(e._ranges_known_rejected())"],
|
|
cwd=str(Path(__file__).resolve().parents[1]),
|
|
capture_output=True, text=True, timeout=60,
|
|
)
|
|
assert out.stdout.strip() == "True", out.stderr
|