mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-10-04 06:15:09 +00:00
ChuckandClaude Opus 5.5
ed0753c1ca
Six small savings with no behaviour change: the odds fetch no longer pretty-prints every response for a debug line; the scroll integer-slice path drops a redundant full-frame np.ascontiguousarray; ledmatrix-web.service gets MALLOC_ARENA_MAX=2 like the display unit; core ESPN responses are parsed via response_json (orjson when installed); and the scroll frame stats go to INFO only for degraded windows plus a 5-minute heartbeat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
420 lines
17 KiB
Python
420 lines
17 KiB
Python
"""
|
|
API Helper
|
|
|
|
HTTP requests, response caching and ESPN fetch helpers for plugins
|
|
(``from src.common import APIHelper``), plus the headers every core request
|
|
sends (:data:`USER_AGENT`, :data:`DEFAULT_HTTP_HEADERS`).
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
from datetime import datetime
|
|
from types import MappingProxyType
|
|
from src.common.espn_dates import (
|
|
ESPN_MAX_LIMIT,
|
|
espn_scoreboard_cache_key,
|
|
read_espn_scoreboard_cache,
|
|
store_espn_scoreboard_cache,
|
|
)
|
|
from src.common.fetch_service import fetch_get, fetch_post, share_connection_pool
|
|
from src.common.json_body import response_json
|
|
from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, cast
|
|
|
|
import requests
|
|
from urllib3.util.retry import Retry
|
|
|
|
if TYPE_CHECKING:
|
|
# What Session() puts in .headers; the stubs only promise a MutableMapping.
|
|
from requests.structures import CaseInsensitiveDict
|
|
|
|
|
|
#: The User-Agent core sends to ESPN and other data APIs. It names the client
|
|
#: and links to it: around 2026-08-04 ESPN began 403ing bare custom tokens
|
|
#: (and browser strings), and this form is what it accepts.
|
|
USER_AGENT = 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)'
|
|
|
|
#: Base headers for core's JSON API requests. Read-only; pass
|
|
#: ``{**DEFAULT_HTTP_HEADERS, ...}`` to add to it. There is deliberately no
|
|
#: Accept-Encoding: requests advertises only what urllib3 can decode here
|
|
#: (``br`` needs the optional brotli package, which is not a requirement), so a
|
|
#: hand-set ``br`` invites a body the client cannot read.
|
|
DEFAULT_HTTP_HEADERS: Mapping[str, str] = MappingProxyType({
|
|
'User-Agent': USER_AGENT,
|
|
'Accept': 'application/json',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
})
|
|
|
|
|
|
class APIHelper:
|
|
"""
|
|
HTTP requests with retries, response caching and ESPN helpers.
|
|
|
|
- Requests go through one ``requests.Session`` that retries GET, HEAD
|
|
and OPTIONS on 429 and 5xx with exponential backoff, and sends
|
|
:data:`DEFAULT_HTTP_HEADERS`. Its connection pool is shared with every
|
|
other helper using the same retry policy, and requests go through the
|
|
core fetch service (``src/common/fetch_service.py``): identical GETs in
|
|
flight are merged, hosts with a budget are paced, and requests are
|
|
counted per plugin. Return values and errors are unchanged.
|
|
- Consecutive requests from one helper are spaced at least
|
|
``set_rate_limit()`` seconds apart (1 second by default). A cache hit
|
|
does not count.
|
|
- With a ``cache_manager``, :meth:`get` caches the parsed JSON under
|
|
``cache_key`` for ``cache_ttl`` seconds. The lifetime is stored with
|
|
the entry, so CacheManager honours it on every later read, whatever
|
|
max_age that read asks for.
|
|
- Failed requests are logged and return None; nothing here raises for a
|
|
network or HTTP error.
|
|
"""
|
|
|
|
def __init__(self, cache_manager=None, default_timeout: int = 30,
|
|
max_retries: int = 3, logger: Optional[logging.Logger] = None):
|
|
"""
|
|
Initialize the APIHelper.
|
|
|
|
Args:
|
|
cache_manager: Optional cache manager for response caching
|
|
default_timeout: Default timeout for requests in seconds
|
|
max_retries: Maximum number of retry attempts
|
|
logger: Optional logger instance
|
|
"""
|
|
self.cache_manager = cache_manager
|
|
self.default_timeout = default_timeout
|
|
self.max_retries = max_retries
|
|
self.logger = logger or logging.getLogger(__name__)
|
|
|
|
# Setup session with retry strategy
|
|
self.session = requests.Session()
|
|
retry_strategy = Retry(
|
|
total=max_retries,
|
|
backoff_factor=1,
|
|
status_forcelist=[429, 500, 502, 503, 504],
|
|
allowed_methods=["GET", "HEAD", "OPTIONS"]
|
|
)
|
|
# The shared adapter for this retry policy: the same retries as a
|
|
# private HTTPAdapter(max_retries=retry_strategy), with the connection
|
|
# pool shared by every helper (fetch_service).
|
|
share_connection_pool(self.session, retry_strategy)
|
|
|
|
self.session.headers.update({**DEFAULT_HTTP_HEADERS, 'Connection': 'keep-alive'})
|
|
|
|
# Rate limiting
|
|
self._last_request_time: float = 0 # wall clock, reported by get_request_stats()
|
|
# The interval is measured on time.monotonic(): a wall-clock step
|
|
# back (NTP correcting a Pi with no RTC) made time_since_last
|
|
# negative and the "remaining interval" sleep as long as the step.
|
|
self._last_request_monotonic: Optional[float] = None
|
|
self._min_request_interval = 1.0 # Minimum seconds between requests
|
|
|
|
def get(self, url: str, params: Optional[Dict] = None,
|
|
headers: Optional[Dict] = None, timeout: Optional[int] = None,
|
|
cache_key: Optional[str] = None, cache_ttl: int = 3600) -> Optional[Dict]:
|
|
"""
|
|
Make a GET request with optional caching.
|
|
|
|
Args:
|
|
url: URL to request
|
|
params: Query parameters
|
|
headers: Additional headers
|
|
timeout: Request timeout (uses default if None)
|
|
cache_key: Key for caching response
|
|
cache_ttl: Cache time-to-live in seconds
|
|
|
|
Returns:
|
|
Response data as dictionary or None if request fails
|
|
"""
|
|
return self._get(url, params, headers, timeout, cache_key, cache_ttl,
|
|
cache_ttl if cache_key else None)
|
|
|
|
def _get(self, url: str, params: Optional[Dict], headers: Optional[Dict],
|
|
timeout: Optional[int], cache_key: Optional[str], cache_ttl: int,
|
|
cache_max_age: Optional[float]) -> Optional[Dict]:
|
|
""":meth:`get`, saying how old a response the fetch service's short
|
|
response cache may hand back (``cache_max_age``, the caller's TTL)."""
|
|
if cache_key and self.cache_manager:
|
|
cached = self._get_from_cache(cache_key, cache_ttl)
|
|
if cached is not None:
|
|
self.logger.debug(f"Using cached response for {cache_key}")
|
|
return cast(Dict[Any, Any], cached)
|
|
|
|
# Rate limiting
|
|
self._enforce_rate_limit()
|
|
|
|
try:
|
|
# Prepare request
|
|
request_headers = cast('CaseInsensitiveDict[Any]', self.session.headers).copy()
|
|
if headers:
|
|
request_headers.update(headers)
|
|
|
|
# Make request
|
|
response = fetch_get(
|
|
self.session,
|
|
url,
|
|
params=params,
|
|
headers=request_headers,
|
|
timeout=timeout or self.default_timeout,
|
|
cache_max_age=cache_max_age,
|
|
)
|
|
response.raise_for_status()
|
|
|
|
# Parse JSON response
|
|
data: Dict[Any, Any] = response_json(response)
|
|
|
|
# Cache response if cache key provided
|
|
if cache_key and self.cache_manager:
|
|
self._set_cache(cache_key, data, cache_ttl)
|
|
|
|
self.logger.debug(f"Successfully fetched {url}")
|
|
return data
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
self.logger.error(f"Request failed for {url}: {e}")
|
|
return None
|
|
|
|
def fetch_espn_scoreboard(self, sport: str, league: str,
|
|
date: Optional[str] = None,
|
|
cache_key: Optional[str] = None,
|
|
cache_ttl: int = 300) -> Optional[Dict]:
|
|
"""
|
|
Fetch ESPN scoreboard data for a specific sport and league.
|
|
|
|
Args:
|
|
sport: Sport name (e.g., 'basketball', 'football')
|
|
league: League name (e.g., 'nba', 'nfl')
|
|
date: Date in YYYYMMDD format (defaults to today)
|
|
cache_key: Cache key for response. By default the canonical
|
|
``espn_scoreboard_cache_key(sport, league, date)``, shared
|
|
with every other consumer of this scoreboard, with the key
|
|
this used before (``espn_{sport}_{league}_{date}``) read as a
|
|
fallback for one release. An explicit key works as before.
|
|
cache_ttl: Cache time-to-live in seconds. A shared entry is
|
|
returned only while it is at most this old.
|
|
|
|
Returns:
|
|
ESPN API response data or None if request fails
|
|
"""
|
|
if date is None:
|
|
date = datetime.now().strftime('%Y%m%d')
|
|
|
|
# Build URL
|
|
url = f"https://site.api.espn.com/apis/site/v2/sports/{sport}/{league}/scoreboard"
|
|
|
|
# Set parameters
|
|
# limit above 500 makes ESPN truncate instead of erroring: college
|
|
# football came back with 25 of 68 games. See src/common/espn_dates.py.
|
|
params = {
|
|
'dates': date,
|
|
'limit': ESPN_MAX_LIMIT
|
|
}
|
|
|
|
if cache_key is not None:
|
|
return self.get(url, params=params, cache_key=cache_key, cache_ttl=cache_ttl)
|
|
|
|
legacy_key = f"espn_{sport}_{league}_{date}"
|
|
try:
|
|
shared_key = espn_scoreboard_cache_key(sport, league, date)
|
|
except ValueError:
|
|
# Not a path or date the canonical key covers: the old key.
|
|
return self.get(url, params=params, cache_key=legacy_key, cache_ttl=cache_ttl)
|
|
if self.cache_manager:
|
|
cached = read_espn_scoreboard_cache(
|
|
self.cache_manager, shared_key, cache_ttl, legacy_keys=(legacy_key,))
|
|
if cached is not None:
|
|
self.logger.debug(f"Using cached response for {shared_key}")
|
|
return cast(Dict[Any, Any], cached)
|
|
data = self._get(url, params, None, None, None, cache_ttl, cache_ttl)
|
|
if data is not None and self.cache_manager:
|
|
store_espn_scoreboard_cache(self.cache_manager, shared_key, data)
|
|
return data
|
|
|
|
def fetch_espn_standings(self, sport: str, league: str,
|
|
cache_key: Optional[str] = None,
|
|
cache_ttl: int = 3600) -> Optional[Dict]:
|
|
"""
|
|
Fetch ESPN standings data for a specific sport and league.
|
|
|
|
Args:
|
|
sport: Sport name
|
|
league: League name
|
|
cache_key: Cache key for response
|
|
cache_ttl: Cache time-to-live in seconds
|
|
|
|
Returns:
|
|
ESPN standings data or None if request fails
|
|
"""
|
|
url = f"https://site.api.espn.com/apis/site/v2/sports/{sport}/{league}/standings"
|
|
|
|
if cache_key is None:
|
|
cache_key = f"espn_standings_{sport}_{league}"
|
|
|
|
return self.get(url, cache_key=cache_key, cache_ttl=cache_ttl)
|
|
|
|
def fetch_espn_rankings(self, sport: str, league: str,
|
|
cache_key: Optional[str] = None,
|
|
cache_ttl: int = 3600) -> Optional[Dict]:
|
|
"""
|
|
Fetch ESPN rankings data for a specific sport and league.
|
|
|
|
Args:
|
|
sport: Sport name
|
|
league: League name
|
|
cache_key: Cache key for response
|
|
cache_ttl: Cache time-to-live in seconds
|
|
|
|
Returns:
|
|
ESPN rankings data or None if request fails
|
|
"""
|
|
url = f"https://site.api.espn.com/apis/site/v2/sports/{sport}/{league}/rankings"
|
|
|
|
if cache_key is None:
|
|
cache_key = f"espn_rankings_{sport}_{league}"
|
|
|
|
return self.get(url, cache_key=cache_key, cache_ttl=cache_ttl)
|
|
|
|
def post(self, url: str, data: Optional[Dict] = None,
|
|
json_data: Optional[Dict] = None,
|
|
headers: Optional[Dict] = None,
|
|
timeout: Optional[int] = None) -> Optional[Dict]:
|
|
"""
|
|
Make a POST request.
|
|
|
|
Args:
|
|
url: URL to request
|
|
data: Form data
|
|
json_data: JSON data
|
|
headers: Additional headers
|
|
timeout: Request timeout
|
|
|
|
Returns:
|
|
Response data as dictionary or None if request fails
|
|
"""
|
|
self._enforce_rate_limit()
|
|
|
|
try:
|
|
request_headers = cast('CaseInsensitiveDict[Any]', self.session.headers).copy()
|
|
if headers:
|
|
request_headers.update(headers)
|
|
|
|
response = fetch_post(
|
|
self.session,
|
|
url,
|
|
data=data,
|
|
json=json_data,
|
|
headers=request_headers,
|
|
timeout=timeout or self.default_timeout
|
|
)
|
|
response.raise_for_status()
|
|
|
|
return cast(Optional[Dict[Any, Any]], response_json(response))
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
self.logger.error(f"POST request failed for {url}: {e}")
|
|
return None
|
|
|
|
def set_cache(self, key: str, data: Any, ttl: int = 3600) -> None:
|
|
"""
|
|
Set cache data.
|
|
|
|
Args:
|
|
key: Cache key
|
|
data: Data to cache
|
|
ttl: Seconds the entry stays valid. Stored with the entry, so
|
|
it applies to every later read of ``key``.
|
|
"""
|
|
self._set_cache(key, data, ttl)
|
|
|
|
def get_cache(self, key: str) -> Optional[Any]:
|
|
"""
|
|
Get cached data.
|
|
|
|
Args:
|
|
key: Cache key
|
|
|
|
Returns:
|
|
Cached data, or None if there is none or it has expired. An
|
|
entry written with a ttl (set_cache, get) expires after that ttl;
|
|
one written without expires after CacheManager's default max_age.
|
|
"""
|
|
return self._get_from_cache(key)
|
|
|
|
def clear_cache(self, pattern: Optional[str] = None) -> None:
|
|
"""
|
|
Clear cache data.
|
|
|
|
Uses CacheManager's clear_cache(), or list_cache_files() and delete()
|
|
for a pattern. A cache manager without those methods is left alone.
|
|
|
|
Args:
|
|
pattern: Optional substring to match cache keys; only matching
|
|
entries are deleted.
|
|
"""
|
|
if not self.cache_manager:
|
|
return
|
|
if pattern:
|
|
if (hasattr(self.cache_manager, 'list_cache_files')
|
|
and hasattr(self.cache_manager, 'delete')):
|
|
for entry in self.cache_manager.list_cache_files():
|
|
key = entry.get('key') if isinstance(entry, dict) else None
|
|
if key and pattern in key:
|
|
self.cache_manager.delete(key)
|
|
else:
|
|
self.logger.debug(
|
|
"Cache manager lacks list_cache_files/delete; "
|
|
"cannot clear by pattern")
|
|
elif hasattr(self.cache_manager, 'clear_cache'):
|
|
self.cache_manager.clear_cache()
|
|
else:
|
|
self.logger.debug("Cache manager exposes no clear method; no-op")
|
|
|
|
def _get_from_cache(self, key: str, max_age: Optional[int] = None) -> Optional[Any]:
|
|
"""Cached data for ``key``, or None. ``max_age`` only matters for an
|
|
entry stored without a ttl; one stored with a ttl uses that."""
|
|
if not self.cache_manager:
|
|
return None
|
|
if max_age is None:
|
|
return self.cache_manager.get(key)
|
|
return self.cache_manager.get(key, max_age=max_age)
|
|
|
|
def _set_cache(self, key: str, data: Any, ttl: Optional[int]) -> None:
|
|
"""Store ``data`` under ``key`` for ``ttl`` seconds."""
|
|
if self.cache_manager:
|
|
self.cache_manager.set(key, data, ttl=ttl)
|
|
|
|
def _enforce_rate_limit(self) -> None:
|
|
"""Enforce rate limiting between requests."""
|
|
if self._last_request_monotonic is not None:
|
|
time_since_last = time.monotonic() - self._last_request_monotonic
|
|
|
|
if time_since_last < self._min_request_interval:
|
|
sleep_time = self._min_request_interval - time_since_last
|
|
time.sleep(sleep_time)
|
|
|
|
self._last_request_monotonic = time.monotonic()
|
|
self._last_request_time = time.time()
|
|
|
|
def set_rate_limit(self, min_interval: float) -> None:
|
|
"""
|
|
Set minimum interval between requests.
|
|
|
|
Args:
|
|
min_interval: Minimum seconds between requests
|
|
"""
|
|
self._min_request_interval = min_interval
|
|
self.logger.debug(f"Rate limit set to {min_interval} seconds")
|
|
|
|
def get_request_stats(self) -> Dict[str, Any]:
|
|
"""
|
|
Get request statistics.
|
|
|
|
Returns:
|
|
Dictionary with request statistics
|
|
"""
|
|
return {
|
|
'min_request_interval': self._min_request_interval,
|
|
'last_request_time': self._last_request_time,
|
|
'time_since_last_request': (
|
|
time.monotonic() - self._last_request_monotonic
|
|
if self._last_request_monotonic is not None
|
|
else time.time() - self._last_request_time),
|
|
}
|