rick/LEDMatrix
1
0
Fork 0
mirror of https://github.com/ChuckBuilds/LEDMatrix.git synced 2026-10-04 14:25:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(logos): one hardened logo download path; shared, real HTTP headers (#612)

Browse Source 
* fix(logos): harden the plugin logo download and share core HTTP headers

download_missing_logo / LogoDownloader.download_logo, the path the
scoreboard plugins use, read response.content with no size cap and wrote
straight to the final path, so a failed or corrupt download could be left
in place and cached as the logo. It now goes through fetch_logo: streamed
with a 10 MB cap, image/* only, decoded by Pillow, converted to RGBA once,
and moved into place atomically. A failure leaves no partial or temp file
and keeps any logo already on disk. LogoHelper._download_logo delegates to
the same code. Public signatures and return values are unchanged; saved
files are pixel-identical to before (RGBA, palette+tRNS, L+tRNS, LA, JPEG).

download_missing_logo reuses one downloader per thread instead of a new
Session per logo. Per thread rather than behind a lock: Session is not
documented thread-safe, and a lock would serialise every plugin's
downloads behind the slowest one.

Placeholders are written atomically, without the test_write.tmp probe.

The logo downloader and background data service now send the real
ChuckBuilds User-Agent from src.common.api_helper (USER_AGENT,
DEFAULT_HTTP_HEADERS) instead of a yourusername/contact@example.com
placeholder, and no longer hand-set Accept-Encoding: br (brotli is not
installed).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(http): drop APIHelper's hand-set brotli encoding; LogoHelper sends the real UA

APIHelper advertised `br` though brotli isn't installed, so a server that
honoured it would send a body requests can't decode. LogoHelper sent a bare
`LEDMatrix-Common/1.0`, the kind of User-Agent ESPN has been rejecting.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chuck
2026-09-23 12:58:11 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f813ea2117
commit 3f8edf5113
8 changed files with 622 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/background_data_service.py
+4 -8
View File
@@ -172,14 +172,10 @@ class BackgroundDataService:
self.session.mount('http://', requests.adapters.HTTPAdapter(max_retries=3))
self.session.mount('https://', requests.adapters.HTTPAdapter(max_retries=3))
# Default headers
self.default_headers = {
'User-Agent': 'LEDMatrix/1.0 (https://github.com/yourusername/LEDMatrix)',
'Accept': 'application/json',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive'
}
# Default headers: core's shared set (real User-Agent, no hand-set
# Accept-Encoding) -- see src/common/api_helper.py.
from src.common.api_helper import DEFAULT_HTTP_HEADERS
self.default_headers = dict(DEFAULT_HTTP_HEADERS)
logger.info(f"BackgroundDataService initialized with {max_workers} workers")
src/common/api_helper.py
+20 -5
View File
@@ -8,14 +8,32 @@ Extracted from LEDMatrix core to provide reusable functionality for plugins.
import logging
import time
from datetime import datetime
from types import MappingProxyType
from src.common.espn_dates import ESPN_MAX_LIMIT
from typing import Any, Dict, Optional
from typing import Any, Dict, Mapping, Optional
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
#: The User-Agent core sends to ESPN and other data APIs. It names the client
#: and links to it: around 2026-08-04 ESPN began 403ing bare custom tokens
#: (and browser strings), and this form is what it accepts.
USER_AGENT = 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)'
#: Base headers for core's JSON API requests. Read-only; pass
#: ``{**DEFAULT_HTTP_HEADERS, ...}`` to add to it. There is deliberately no
#: Accept-Encoding: requests advertises only what urllib3 can decode here
#: (``br`` needs the optional brotli package, which is not a requirement), so a
#: hand-set ``br`` invites a body the client cannot read.
DEFAULT_HTTP_HEADERS: Mapping[str, str] = MappingProxyType({
'User-Agent': USER_AGENT,
'Accept': 'application/json',
'Accept-Language': 'en-US,en;q=0.9',
})
class APIHelper:
"""
Helper class for HTTP requests, caching, and ESPN API integration.
@@ -57,12 +75,9 @@ class APIHelper:
# Default headers
self.session.headers.update({
# Identifies the client and links to it: ESPN began 403ing bare
# custom tokens (and browser strings) around 2026-08-04.
'User-Agent': 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)',
'User-Agent': USER_AGENT,
'Accept': 'application/json',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive'
})
src/common/logo_helper.py
+15 -58
View File
@@ -6,19 +6,16 @@ Extracted from LEDMatrix core to provide reusable functionality for plugins.
"""
import logging
import os
import tempfile
import time
from pathlib import Path
from typing import Dict, List, Optional, Union
import requests
from PIL import Image
from src.common.api_helper import USER_AGENT
from src.common.permission_utils import (
ensure_directory_permissions,
ensure_file_permissions,
get_assets_dir_mode,
get_assets_file_mode
)
# How long a missing logo stays remembered as missing.
@@ -61,6 +58,7 @@ def _usable_scale(scale) -> float:
# Well above any real team logo; bounds what a remote URL can write to disk.
# The cap for every logo download: src.logo_downloader.fetch_logo uses it too.
MAX_LOGO_BYTES = 10 * 1024 * 1024
@@ -107,7 +105,7 @@ class LogoHelper:
# Session for HTTP requests
self.session = requests.Session()
self.session.headers.update({
'User-Agent': 'LEDMatrix-Common/1.0',
'User-Agent': USER_AGENT,
'Accept': 'image/*',
})
@@ -408,64 +406,23 @@ class LogoHelper:
self._cache_order.append(cache_key)
def _download_logo(self, url: str, file_path: Path) -> None:
"""Download logo from URL.
"""Download a logo from ``url`` to ``file_path``; raises on failure.
The response size is capped and the saved file is verified as a
decodable image before it is left on disk: a logo URL is remote
input, and without this an oversized or malformed response would
be cached for every later load_logo() call to trip over.
Delegates to ``src.logo_downloader.fetch_logo``, the same hardened
download the scoreboard plugins use: streamed and capped at
``MAX_LOGO_BYTES``, ``image/*`` only, decoded by Pillow, stored as an
RGBA PNG, and moved into place atomically -- a failure leaves neither
a partial file nor a temp file. Uses this helper's own session.
The body is streamed and counted as it arrives rather than read
through response.content, which buffers the whole thing first —
a server that omits Content-Length and never stops sending would
exhaust memory before any size check could run. Nothing lands at
file_path until the download completes and decodes, so a failed
download cannot leave a truncated logo behind either.
Imported lazily: src.logo_downloader imports src.common, so a
module-level import here would be circular.
"""
from src.logo_downloader import fetch_logo
# Ensure directory exists with proper permissions
ensure_directory_permissions(file_path.parent, get_assets_dir_mode())
# A unique temp name, not a fixed "<name>.part": two plugins can
# ask for the same logo at once, and a shared name would let them
# interleave writes into one file, publish the mixture, or delete
# each other's partial. Same directory, so os.replace stays atomic.
fd, tmp_name = tempfile.mkstemp(
dir=str(file_path.parent), prefix=file_path.name + '.', suffix='.part')
tmp_path = Path(tmp_name)
try:
# fdopen outermost so the descriptor mkstemp handed back is
# always adopted and closed, including when the request itself
# raises — load_logo_with_download swallows that, so a leak
# here would accumulate quietly on a URL that keeps failing.
with os.fdopen(fd, 'wb') as f:
with self.session.get(url, timeout=30, stream=True) as response:
response.raise_for_status()
downloaded = 0
for chunk in response.iter_content(chunk_size=64 * 1024):
if not chunk:
continue
downloaded += len(chunk)
if downloaded > MAX_LOGO_BYTES:
raise ValueError(
f"Logo at {url} exceeds the "
f"{MAX_LOGO_BYTES}-byte limit; not saved")
f.write(chunk)
# Verify it decodes before it becomes the cached logo. PIL
# raises DecompressionBombError past its own pixel limit; a
# partial or non-image response raises UnidentifiedImageError
# (an OSError subclass).
with Image.open(tmp_path) as probe:
probe.load()
os.replace(tmp_path, file_path)
except BaseException:
tmp_path.unlink(missing_ok=True)
raise
# Set proper file permissions after saving
ensure_file_permissions(file_path, get_assets_file_mode())
fetch_logo(self.session, url, file_path, timeout=30,
max_bytes=MAX_LOGO_BYTES)
self.logger.debug(f"Downloaded logo to {file_path}")
def _create_placeholder_logo(self, team_abbr: str,
src/logo_downloader.py
+195 -75
View File
@@ -7,17 +7,21 @@ with special support for FCS teams and other NCAA divisions.
import os
import re
import tempfile
import threading
import time
import logging
import requests
import json
from typing import Dict, List, Optional, Tuple
from pathlib import Path
from PIL import Image, ImageDraw, ImageFont
from PIL import Image, ImageDraw, ImageFont, UnidentifiedImageError
from src.common.font_layout import load_truetype
from PIL.PngImagePlugin import PngInfo
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from src.common.api_helper import DEFAULT_HTTP_HEADERS
from src.common.logo_helper import MAX_LOGO_BYTES
from src.common.permission_utils import (
ensure_directory_permissions,
ensure_file_permissions,
@@ -27,6 +31,142 @@ from src.common.permission_utils import (
logger = logging.getLogger(__name__)
#: Accept header for logo image requests (the JSON default is for the API).
LOGO_ACCEPT = 'image/png,image/*;q=0.8'
def _is_image_content_type(content_type: str) -> bool:
"""True for an ``image/*`` media type, ignoring parameters and case.
Only a cheap gate before the body is read; Pillow decoding the bytes is
what actually decides whether they are an image.
"""
return content_type.split(';', 1)[0].strip().lower().startswith('image/')
def _to_rgba(img: Image.Image) -> Image.Image:
"""``img`` as RGBA, keeping its transparency.
One conversion covers every mode: Pillow folds a palette's or a
greyscale/RGB image's ``transparency`` entry into the alpha channel when
converting to RGBA, and an image without one gets an opaque alpha. Plugins
paste logos with the image as its own mask, so the alpha is the part that
has to survive.
"""
return img.copy() if img.mode == 'RGBA' else img.convert('RGBA')
def _publish(tmp_path: Path, filepath: Path) -> None:
"""Give a finished temp file the asset mode, then move it into place.
``mkstemp`` creates files 0600, so the mode is set before the rename: the
logo must never be visible under its real name unreadable to the web
service's user. ``os.replace`` within one directory is atomic, so a reader
sees the old file or the new one, never a half-written one.
"""
ensure_file_permissions(tmp_path, get_assets_file_mode())
os.replace(tmp_path, filepath)
def _temp_beside(filepath: Path) -> Tuple[int, Path]:
"""A unique temp file in ``filepath``'s directory.
Unique rather than a fixed ``<name>.part`` because two plugins can ask for
the same logo at once; a shared name would let them interleave writes into
one file, or delete each other's partial. The same directory keeps
``os.replace`` atomic.
"""
fd, tmp_name = tempfile.mkstemp(
dir=str(filepath.parent), prefix=filepath.name + '.', suffix='.part')
return fd, Path(tmp_name)
def save_png_atomically(image: Image.Image, filepath: Path,
pnginfo: Optional[PngInfo] = None) -> None:
"""Save ``image`` as a PNG at ``filepath`` without ever exposing a partial file.
Raises on failure (including PermissionError for an unwritable
directory), leaving any previous file at ``filepath`` untouched and no
temp file behind.
"""
filepath = Path(filepath)
fd, tmp_path = _temp_beside(filepath)
try:
with os.fdopen(fd, 'wb') as f:
if pnginfo is not None:
image.save(f, 'PNG', pnginfo=pnginfo)
else:
image.save(f, 'PNG')
_publish(tmp_path, filepath)
except BaseException:
tmp_path.unlink(missing_ok=True)
raise
def fetch_logo(session: requests.Session, url: str, filepath: Path, *,
headers: Optional[Dict[str, str]] = None, timeout: float = 30,
max_bytes: Optional[int] = None) -> None:
"""Download the image at ``url`` and save it at ``filepath`` as an RGBA PNG.
The one hardened logo download; ``LogoDownloader.download_logo`` and
``LogoHelper._download_logo`` both go through it. A logo URL is remote
input, and whatever lands at ``filepath`` is cached and loaded on every
later frame, so nothing is written there until the bytes have been
size-checked and decoded:
- the response must be ``image/*`` and is streamed, counted as it
arrives, and abandoned past ``max_bytes`` (``MAX_LOGO_BYTES`` by
default) -- ``response.content`` would buffer a body that never ends
before any check could run;
- the bytes go to a unique temp file beside ``filepath`` and must decode
with Pillow (which also enforces its decompression-bomb limit);
- the image is converted to RGBA once, rewritten as PNG, and moved into
place atomically.
Raises on any failure, with no temp file left and any previous file at
``filepath`` intact. The caller creates the directory.
"""
if max_bytes is None:
max_bytes = MAX_LOGO_BYTES
filepath = Path(filepath)
fd, tmp_path = _temp_beside(filepath)
try:
# fdopen outermost so the descriptor is adopted and closed even when
# the request itself raises.
with os.fdopen(fd, 'wb') as f:
with session.get(url, headers=headers, timeout=timeout,
stream=True) as response:
response.raise_for_status()
content_type = response.headers.get('content-type') or ''
if not _is_image_content_type(content_type):
raise ValueError(
f"Logo at {url} is not an image "
f"(content-type {content_type!r}); not saved")
received = 0
for chunk in response.iter_content(chunk_size=64 * 1024):
if not chunk:
continue
received += len(chunk)
if received > max_bytes:
raise ValueError(
f"Logo at {url} exceeds the "
f"{max_bytes}-byte limit; not saved")
f.write(chunk)
# UnidentifiedImageError (an OSError) for bytes that are not an
# image, DecompressionBombError past Pillow's pixel limit, OSError for
# a truncated one.
with Image.open(tmp_path) as img:
img.load()
rgba = _to_rgba(img)
with open(tmp_path, 'wb') as f:
rgba.save(f, 'PNG')
_publish(tmp_path, filepath)
except BaseException:
tmp_path.unlink(missing_ok=True)
raise
#: PNG text key stamped into a generated placeholder so a later run can tell it
#: apart from a real logo that happens to be small.
PLACEHOLDER_MARKER = "ledmatrix_placeholder"
@@ -199,14 +339,8 @@ class LogoDownloader:
self.session.mount("https://", adapter)
self.session.mount("http://", adapter)
# Set up headers
self.headers = {
'User-Agent': 'LEDMatrix/1.0 (https://github.com/yourusername/LEDMatrix; contact@example.com)',
'Accept': 'application/json',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive'
}
# Core's shared API headers; a plain dict so callers may adjust theirs.
self.headers = dict(DEFAULT_HTTP_HEADERS)
@staticmethod
def normalize_abbreviation(abbreviation: str) -> str:
@@ -301,50 +435,19 @@ class LogoDownloader:
return False
def download_logo(self, logo_url: str, filepath: Path, team_abbreviation: str) -> bool:
"""Download a single logo from URL and save to filepath."""
"""Download a single logo from URL and save it to filepath as an RGBA PNG.
Returns False (and logs why) on any failure; see ``fetch_logo`` for
the guarantees -- in particular a failure never leaves a partial file,
and never replaces a logo already at ``filepath``.
"""
filepath = Path(filepath)
try:
response = self.session.get(logo_url, headers=self.headers, timeout=self.request_timeout)
response.raise_for_status()
# Verify it's actually an image
content_type = response.headers.get('content-type', '').lower()
if not any(img_type in content_type for img_type in ['image/png', 'image/jpeg', 'image/jpg', 'image/gif']):
logger.warning(f"Downloaded content for {team_abbreviation} is not an image: {content_type}")
return False
with open(filepath, 'wb') as f:
f.write(response.content)
# Verify and convert the downloaded image to RGBA format
try:
with Image.open(filepath) as img:
# Convert to RGBA to avoid PIL warnings about palette images with transparency
if img.mode in ('P', 'LA', 'L'):
# Convert palette or grayscale images to RGBA
img = img.convert('RGBA')
elif img.mode == 'RGB':
# Convert RGB to RGBA (add alpha channel)
img = img.convert('RGBA')
elif img.mode != 'RGBA':
# For any other mode, convert to RGBA
img = img.convert('RGBA')
# Save the converted image
img.save(filepath, 'PNG')
# Set proper file permissions after saving
ensure_file_permissions(filepath, get_assets_file_mode())
logger.info(f"Successfully downloaded and converted logo for {team_abbreviation} -> {filepath.name}")
return True
except Exception as e:
logger.error(f"Downloaded file for {team_abbreviation} is not a valid image or conversion failed: {e}")
try:
os.remove(filepath) # Remove invalid file
except OSError:
pass
return False
fetch_logo(self.session, logo_url, filepath,
headers={**self.headers, 'Accept': LOGO_ACCEPT},
timeout=self.request_timeout)
logger.info(f"Successfully downloaded and converted logo for {team_abbreviation} -> {filepath.name}")
return True
except PermissionError as e:
logger.error(f"Permission denied downloading logo for {team_abbreviation}: {e}")
logger.error("Please run: sudo ./scripts/fix_perms/fix_assets_permissions.sh")
@@ -352,10 +455,13 @@ class LogoDownloader:
except requests.exceptions.RequestException as e:
logger.error(f"Failed to download logo for {team_abbreviation}: {e}")
return False
except (ValueError, UnidentifiedImageError, Image.DecompressionBombError) as e:
logger.error(f"Rejected downloaded logo for {team_abbreviation}: {e}")
return False
except Exception as e:
logger.error(f"Unexpected error downloading logo for {team_abbreviation}: {e}")
return False
# Allowlist for the league_code segment interpolated into ESPN API URLs
_SAFE_LEAGUE_CODE_RE = re.compile(r'^[a-z0-9_-]+$')
@@ -728,20 +834,7 @@ class LogoDownloader:
filename = f"{self.normalize_abbreviation(team_abbreviation)}.png"
filepath = Path(logo_dir) / filename
# Check if we can write to the directory
try:
# Test write permissions by creating a temporary file
test_file = filepath.parent / "test_write.tmp"
test_file.touch()
test_file.unlink() # Remove the test file
except PermissionError:
logger.error(f"Permission denied: Cannot write to directory {logo_dir}")
return False
except Exception as e:
logger.error(f"Directory access error for {logo_dir}: {e}")
return False
# Create a simple placeholder logo
logo = Image.new('RGBA', (64, 64), (100, 100, 100, 255)) # Gray background
draw = ImageDraw.Draw(logo)
@@ -774,14 +867,16 @@ class LogoDownloader:
# proof the logo was fetched.
metadata = PngInfo()
metadata.add_text(PLACEHOLDER_MARKER, str(time.time()))
logo.save(filepath, "PNG", pnginfo=metadata)
# Set proper file permissions after saving
ensure_file_permissions(filepath, get_assets_file_mode())
# Atomic, and it sets the asset mode; an unwritable directory
# surfaces here as PermissionError.
save_png_atomically(logo, filepath, pnginfo=metadata)
logger.info(f"Created placeholder logo for {team_abbreviation} at {filepath}")
return True
except PermissionError as e:
logger.error(f"Permission denied: Cannot write placeholder logo to {logo_dir}: {e}")
return False
except Exception as e:
logger.error(f"Failed to create placeholder logo for {team_abbreviation}: {e}")
return False
@@ -837,6 +932,31 @@ def get_soccer_league_key(league_code: str) -> str:
return f"soccer_{league_code}"
_thread_state = threading.local()
def shared_downloader() -> LogoDownloader:
"""The calling thread's reusable LogoDownloader.
``download_missing_logo`` used to build a new downloader -- a new
``requests.Session``, retry adapter and connection pool -- for every logo.
Reusing one keeps connections to ESPN's CDN alive between logos.
One per thread rather than one behind a lock: ``requests.Session`` is not
documented as safe for concurrent use, and plugins download on worker
threads (football-scoreboard runs a small pool precisely so downloads
overlap). A lock would serialise every plugin's downloads behind the
slowest one -- up to 30s per attempt, with retries. Pool threads persist,
so each still reuses its own session; a thread's downloader goes with the
thread.
"""
downloader = getattr(_thread_state, 'downloader', None)
if downloader is None:
downloader = LogoDownloader()
_thread_state.downloader = downloader
return downloader
# Convenience function for easy integration
def download_missing_logo(league: str, team_id: str, team_abbreviation: str, logo_path: Path, logo_url: str | None = None, create_placeholder: bool = True) -> bool:
"""
@@ -852,8 +972,8 @@ def download_missing_logo(league: str, team_id: str, team_abbreviation: str, log
Returns:
True if logo exists or was successfully downloaded, False otherwise
"""
downloader = LogoDownloader()
downloader = shared_downloader()
# Use the directory from the logo_path parameter (respects config settings)
logo_path = Path(logo_path)
if not logo_path.is_absolute():
@@ -920,5 +1040,5 @@ def download_all_logos_for_league(league: str, force_download: bool = False) ->
Returns:
Tuple of (downloaded_count, failed_count)
"""
downloader = LogoDownloader()
downloader = shared_downloader()
return downloader.download_missing_logos_for_league(league, force_download)