diff --git a/CHANGELOG.md b/CHANGELOG.md index 36c80b86..8b2d3927 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,35 @@ core, the monorepo or the registry's third-party plugins calls them: instead of handing `config.json` to root, and an install path with "secrets" in a directory name no longer makes `config.json` mode 0640. +New names in existing modules (no new modules; a plugin importing these must +floor on the release that ships them): + +- `src.common.api_helper`: `USER_AGENT`, `DEFAULT_HTTP_HEADERS` (read-only). +- `src.logo_downloader`: `fetch_logo`, `save_png_atomically`, + `shared_downloader`. + +### Logo downloads + +- `download_missing_logo` / `LogoDownloader.download_logo` (the path the + scoreboard plugins use) now stream the logo with a 10 MB cap, accept only an + `image/*` response that Pillow can decode, and move the finished RGBA PNG + into place atomically. A failed, oversized or non-image download no longer + leaves a partial file behind, and no longer replaces a logo already on disk. + `LogoHelper._download_logo` goes through the same code. Signatures and return + values are unchanged; saved files are pixel-identical to before. +- `download_missing_logo` reuses one downloader (one `requests.Session`) per + thread instead of building a new one for every logo. +- Placeholder logos are written atomically, without the `test_write.tmp` + probe file. + +### HTTP headers + +- The logo downloader and the background data service send the real + `LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)` User-Agent + instead of a `yourusername` / `contact@example.com` placeholder, and no + longer set `Accept-Encoding: ... br` by hand (brotli is not installed, so a + `br` response could not be decoded); requests picks the encodings. + ## 3.5.0 New modules a plugin may import via `src.*` (floor on 3.5.0): diff --git a/src/background_data_service.py b/src/background_data_service.py index f12b3750..22989a56 100644 --- a/src/background_data_service.py +++ b/src/background_data_service.py @@ -172,14 +172,10 @@ class BackgroundDataService: self.session.mount('http://', requests.adapters.HTTPAdapter(max_retries=3)) self.session.mount('https://', requests.adapters.HTTPAdapter(max_retries=3)) - # Default headers - self.default_headers = { - 'User-Agent': 'LEDMatrix/1.0 (https://github.com/yourusername/LEDMatrix)', - 'Accept': 'application/json', - 'Accept-Language': 'en-US,en;q=0.9', - 'Accept-Encoding': 'gzip, deflate, br', - 'Connection': 'keep-alive' - } + # Default headers: core's shared set (real User-Agent, no hand-set + # Accept-Encoding) -- see src/common/api_helper.py. + from src.common.api_helper import DEFAULT_HTTP_HEADERS + self.default_headers = dict(DEFAULT_HTTP_HEADERS) logger.info(f"BackgroundDataService initialized with {max_workers} workers") diff --git a/src/common/api_helper.py b/src/common/api_helper.py index 88bdbe11..d025b378 100644 --- a/src/common/api_helper.py +++ b/src/common/api_helper.py @@ -8,14 +8,32 @@ Extracted from LEDMatrix core to provide reusable functionality for plugins. import logging import time from datetime import datetime +from types import MappingProxyType from src.common.espn_dates import ESPN_MAX_LIMIT -from typing import Any, Dict, Optional +from typing import Any, Dict, Mapping, Optional import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry +#: The User-Agent core sends to ESPN and other data APIs. It names the client +#: and links to it: around 2026-08-04 ESPN began 403ing bare custom tokens +#: (and browser strings), and this form is what it accepts. +USER_AGENT = 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)' + +#: Base headers for core's JSON API requests. Read-only; pass +#: ``{**DEFAULT_HTTP_HEADERS, ...}`` to add to it. There is deliberately no +#: Accept-Encoding: requests advertises only what urllib3 can decode here +#: (``br`` needs the optional brotli package, which is not a requirement), so a +#: hand-set ``br`` invites a body the client cannot read. +DEFAULT_HTTP_HEADERS: Mapping[str, str] = MappingProxyType({ + 'User-Agent': USER_AGENT, + 'Accept': 'application/json', + 'Accept-Language': 'en-US,en;q=0.9', +}) + + class APIHelper: """ Helper class for HTTP requests, caching, and ESPN API integration. @@ -57,12 +75,9 @@ class APIHelper: # Default headers self.session.headers.update({ - # Identifies the client and links to it: ESPN began 403ing bare - # custom tokens (and browser strings) around 2026-08-04. - 'User-Agent': 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)', + 'User-Agent': USER_AGENT, 'Accept': 'application/json', 'Accept-Language': 'en-US,en;q=0.9', - 'Accept-Encoding': 'gzip, deflate, br', 'Connection': 'keep-alive' }) diff --git a/src/common/logo_helper.py b/src/common/logo_helper.py index c8a5c221..a574b89c 100644 --- a/src/common/logo_helper.py +++ b/src/common/logo_helper.py @@ -6,19 +6,16 @@ Extracted from LEDMatrix core to provide reusable functionality for plugins. """ import logging -import os -import tempfile import time from pathlib import Path from typing import Dict, List, Optional, Union import requests from PIL import Image +from src.common.api_helper import USER_AGENT from src.common.permission_utils import ( ensure_directory_permissions, - ensure_file_permissions, get_assets_dir_mode, - get_assets_file_mode ) # How long a missing logo stays remembered as missing. @@ -61,6 +58,7 @@ def _usable_scale(scale) -> float: # Well above any real team logo; bounds what a remote URL can write to disk. +# The cap for every logo download: src.logo_downloader.fetch_logo uses it too. MAX_LOGO_BYTES = 10 * 1024 * 1024 @@ -107,7 +105,7 @@ class LogoHelper: # Session for HTTP requests self.session = requests.Session() self.session.headers.update({ - 'User-Agent': 'LEDMatrix-Common/1.0', + 'User-Agent': USER_AGENT, 'Accept': 'image/*', }) @@ -408,64 +406,23 @@ class LogoHelper: self._cache_order.append(cache_key) def _download_logo(self, url: str, file_path: Path) -> None: - """Download logo from URL. + """Download a logo from ``url`` to ``file_path``; raises on failure. - The response size is capped and the saved file is verified as a - decodable image before it is left on disk: a logo URL is remote - input, and without this an oversized or malformed response would - be cached for every later load_logo() call to trip over. + Delegates to ``src.logo_downloader.fetch_logo``, the same hardened + download the scoreboard plugins use: streamed and capped at + ``MAX_LOGO_BYTES``, ``image/*`` only, decoded by Pillow, stored as an + RGBA PNG, and moved into place atomically -- a failure leaves neither + a partial file nor a temp file. Uses this helper's own session. - The body is streamed and counted as it arrives rather than read - through response.content, which buffers the whole thing first — - a server that omits Content-Length and never stops sending would - exhaust memory before any size check could run. Nothing lands at - file_path until the download completes and decodes, so a failed - download cannot leave a truncated logo behind either. + Imported lazily: src.logo_downloader imports src.common, so a + module-level import here would be circular. """ + from src.logo_downloader import fetch_logo + # Ensure directory exists with proper permissions ensure_directory_permissions(file_path.parent, get_assets_dir_mode()) - - # A unique temp name, not a fixed ".part": two plugins can - # ask for the same logo at once, and a shared name would let them - # interleave writes into one file, publish the mixture, or delete - # each other's partial. Same directory, so os.replace stays atomic. - fd, tmp_name = tempfile.mkstemp( - dir=str(file_path.parent), prefix=file_path.name + '.', suffix='.part') - tmp_path = Path(tmp_name) - try: - # fdopen outermost so the descriptor mkstemp handed back is - # always adopted and closed, including when the request itself - # raises — load_logo_with_download swallows that, so a leak - # here would accumulate quietly on a URL that keeps failing. - with os.fdopen(fd, 'wb') as f: - with self.session.get(url, timeout=30, stream=True) as response: - response.raise_for_status() - downloaded = 0 - for chunk in response.iter_content(chunk_size=64 * 1024): - if not chunk: - continue - downloaded += len(chunk) - if downloaded > MAX_LOGO_BYTES: - raise ValueError( - f"Logo at {url} exceeds the " - f"{MAX_LOGO_BYTES}-byte limit; not saved") - f.write(chunk) - - # Verify it decodes before it becomes the cached logo. PIL - # raises DecompressionBombError past its own pixel limit; a - # partial or non-image response raises UnidentifiedImageError - # (an OSError subclass). - with Image.open(tmp_path) as probe: - probe.load() - - os.replace(tmp_path, file_path) - except BaseException: - tmp_path.unlink(missing_ok=True) - raise - - # Set proper file permissions after saving - ensure_file_permissions(file_path, get_assets_file_mode()) - + fetch_logo(self.session, url, file_path, timeout=30, + max_bytes=MAX_LOGO_BYTES) self.logger.debug(f"Downloaded logo to {file_path}") def _create_placeholder_logo(self, team_abbr: str, diff --git a/src/logo_downloader.py b/src/logo_downloader.py index 43b8ff1a..c74096df 100644 --- a/src/logo_downloader.py +++ b/src/logo_downloader.py @@ -7,17 +7,21 @@ with special support for FCS teams and other NCAA divisions. import os import re +import tempfile +import threading import time import logging import requests import json from typing import Dict, List, Optional, Tuple from pathlib import Path -from PIL import Image, ImageDraw, ImageFont +from PIL import Image, ImageDraw, ImageFont, UnidentifiedImageError from src.common.font_layout import load_truetype from PIL.PngImagePlugin import PngInfo from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry +from src.common.api_helper import DEFAULT_HTTP_HEADERS +from src.common.logo_helper import MAX_LOGO_BYTES from src.common.permission_utils import ( ensure_directory_permissions, ensure_file_permissions, @@ -27,6 +31,142 @@ from src.common.permission_utils import ( logger = logging.getLogger(__name__) +#: Accept header for logo image requests (the JSON default is for the API). +LOGO_ACCEPT = 'image/png,image/*;q=0.8' + + +def _is_image_content_type(content_type: str) -> bool: + """True for an ``image/*`` media type, ignoring parameters and case. + + Only a cheap gate before the body is read; Pillow decoding the bytes is + what actually decides whether they are an image. + """ + return content_type.split(';', 1)[0].strip().lower().startswith('image/') + + +def _to_rgba(img: Image.Image) -> Image.Image: + """``img`` as RGBA, keeping its transparency. + + One conversion covers every mode: Pillow folds a palette's or a + greyscale/RGB image's ``transparency`` entry into the alpha channel when + converting to RGBA, and an image without one gets an opaque alpha. Plugins + paste logos with the image as its own mask, so the alpha is the part that + has to survive. + """ + return img.copy() if img.mode == 'RGBA' else img.convert('RGBA') + + +def _publish(tmp_path: Path, filepath: Path) -> None: + """Give a finished temp file the asset mode, then move it into place. + + ``mkstemp`` creates files 0600, so the mode is set before the rename: the + logo must never be visible under its real name unreadable to the web + service's user. ``os.replace`` within one directory is atomic, so a reader + sees the old file or the new one, never a half-written one. + """ + ensure_file_permissions(tmp_path, get_assets_file_mode()) + os.replace(tmp_path, filepath) + + +def _temp_beside(filepath: Path) -> Tuple[int, Path]: + """A unique temp file in ``filepath``'s directory. + + Unique rather than a fixed ``.part`` because two plugins can ask for + the same logo at once; a shared name would let them interleave writes into + one file, or delete each other's partial. The same directory keeps + ``os.replace`` atomic. + """ + fd, tmp_name = tempfile.mkstemp( + dir=str(filepath.parent), prefix=filepath.name + '.', suffix='.part') + return fd, Path(tmp_name) + + +def save_png_atomically(image: Image.Image, filepath: Path, + pnginfo: Optional[PngInfo] = None) -> None: + """Save ``image`` as a PNG at ``filepath`` without ever exposing a partial file. + + Raises on failure (including PermissionError for an unwritable + directory), leaving any previous file at ``filepath`` untouched and no + temp file behind. + """ + filepath = Path(filepath) + fd, tmp_path = _temp_beside(filepath) + try: + with os.fdopen(fd, 'wb') as f: + if pnginfo is not None: + image.save(f, 'PNG', pnginfo=pnginfo) + else: + image.save(f, 'PNG') + _publish(tmp_path, filepath) + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + + +def fetch_logo(session: requests.Session, url: str, filepath: Path, *, + headers: Optional[Dict[str, str]] = None, timeout: float = 30, + max_bytes: Optional[int] = None) -> None: + """Download the image at ``url`` and save it at ``filepath`` as an RGBA PNG. + + The one hardened logo download; ``LogoDownloader.download_logo`` and + ``LogoHelper._download_logo`` both go through it. A logo URL is remote + input, and whatever lands at ``filepath`` is cached and loaded on every + later frame, so nothing is written there until the bytes have been + size-checked and decoded: + + - the response must be ``image/*`` and is streamed, counted as it + arrives, and abandoned past ``max_bytes`` (``MAX_LOGO_BYTES`` by + default) -- ``response.content`` would buffer a body that never ends + before any check could run; + - the bytes go to a unique temp file beside ``filepath`` and must decode + with Pillow (which also enforces its decompression-bomb limit); + - the image is converted to RGBA once, rewritten as PNG, and moved into + place atomically. + + Raises on any failure, with no temp file left and any previous file at + ``filepath`` intact. The caller creates the directory. + """ + if max_bytes is None: + max_bytes = MAX_LOGO_BYTES + filepath = Path(filepath) + fd, tmp_path = _temp_beside(filepath) + try: + # fdopen outermost so the descriptor is adopted and closed even when + # the request itself raises. + with os.fdopen(fd, 'wb') as f: + with session.get(url, headers=headers, timeout=timeout, + stream=True) as response: + response.raise_for_status() + content_type = response.headers.get('content-type') or '' + if not _is_image_content_type(content_type): + raise ValueError( + f"Logo at {url} is not an image " + f"(content-type {content_type!r}); not saved") + received = 0 + for chunk in response.iter_content(chunk_size=64 * 1024): + if not chunk: + continue + received += len(chunk) + if received > max_bytes: + raise ValueError( + f"Logo at {url} exceeds the " + f"{max_bytes}-byte limit; not saved") + f.write(chunk) + + # UnidentifiedImageError (an OSError) for bytes that are not an + # image, DecompressionBombError past Pillow's pixel limit, OSError for + # a truncated one. + with Image.open(tmp_path) as img: + img.load() + rgba = _to_rgba(img) + with open(tmp_path, 'wb') as f: + rgba.save(f, 'PNG') + _publish(tmp_path, filepath) + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + + #: PNG text key stamped into a generated placeholder so a later run can tell it #: apart from a real logo that happens to be small. PLACEHOLDER_MARKER = "ledmatrix_placeholder" @@ -199,14 +339,8 @@ class LogoDownloader: self.session.mount("https://", adapter) self.session.mount("http://", adapter) - # Set up headers - self.headers = { - 'User-Agent': 'LEDMatrix/1.0 (https://github.com/yourusername/LEDMatrix; contact@example.com)', - 'Accept': 'application/json', - 'Accept-Language': 'en-US,en;q=0.9', - 'Accept-Encoding': 'gzip, deflate, br', - 'Connection': 'keep-alive' - } + # Core's shared API headers; a plain dict so callers may adjust theirs. + self.headers = dict(DEFAULT_HTTP_HEADERS) @staticmethod def normalize_abbreviation(abbreviation: str) -> str: @@ -301,50 +435,19 @@ class LogoDownloader: return False def download_logo(self, logo_url: str, filepath: Path, team_abbreviation: str) -> bool: - """Download a single logo from URL and save to filepath.""" + """Download a single logo from URL and save it to filepath as an RGBA PNG. + + Returns False (and logs why) on any failure; see ``fetch_logo`` for + the guarantees -- in particular a failure never leaves a partial file, + and never replaces a logo already at ``filepath``. + """ + filepath = Path(filepath) try: - response = self.session.get(logo_url, headers=self.headers, timeout=self.request_timeout) - response.raise_for_status() - - # Verify it's actually an image - content_type = response.headers.get('content-type', '').lower() - if not any(img_type in content_type for img_type in ['image/png', 'image/jpeg', 'image/jpg', 'image/gif']): - logger.warning(f"Downloaded content for {team_abbreviation} is not an image: {content_type}") - return False - - with open(filepath, 'wb') as f: - f.write(response.content) - - # Verify and convert the downloaded image to RGBA format - try: - with Image.open(filepath) as img: - # Convert to RGBA to avoid PIL warnings about palette images with transparency - if img.mode in ('P', 'LA', 'L'): - # Convert palette or grayscale images to RGBA - img = img.convert('RGBA') - elif img.mode == 'RGB': - # Convert RGB to RGBA (add alpha channel) - img = img.convert('RGBA') - elif img.mode != 'RGBA': - # For any other mode, convert to RGBA - img = img.convert('RGBA') - - # Save the converted image - img.save(filepath, 'PNG') - - # Set proper file permissions after saving - ensure_file_permissions(filepath, get_assets_file_mode()) - - logger.info(f"Successfully downloaded and converted logo for {team_abbreviation} -> {filepath.name}") - return True - except Exception as e: - logger.error(f"Downloaded file for {team_abbreviation} is not a valid image or conversion failed: {e}") - try: - os.remove(filepath) # Remove invalid file - except OSError: - pass - return False - + fetch_logo(self.session, logo_url, filepath, + headers={**self.headers, 'Accept': LOGO_ACCEPT}, + timeout=self.request_timeout) + logger.info(f"Successfully downloaded and converted logo for {team_abbreviation} -> {filepath.name}") + return True except PermissionError as e: logger.error(f"Permission denied downloading logo for {team_abbreviation}: {e}") logger.error("Please run: sudo ./scripts/fix_perms/fix_assets_permissions.sh") @@ -352,10 +455,13 @@ class LogoDownloader: except requests.exceptions.RequestException as e: logger.error(f"Failed to download logo for {team_abbreviation}: {e}") return False + except (ValueError, UnidentifiedImageError, Image.DecompressionBombError) as e: + logger.error(f"Rejected downloaded logo for {team_abbreviation}: {e}") + return False except Exception as e: logger.error(f"Unexpected error downloading logo for {team_abbreviation}: {e}") return False - + # Allowlist for the league_code segment interpolated into ESPN API URLs _SAFE_LEAGUE_CODE_RE = re.compile(r'^[a-z0-9_-]+$') @@ -728,20 +834,7 @@ class LogoDownloader: filename = f"{self.normalize_abbreviation(team_abbreviation)}.png" filepath = Path(logo_dir) / filename - - # Check if we can write to the directory - try: - # Test write permissions by creating a temporary file - test_file = filepath.parent / "test_write.tmp" - test_file.touch() - test_file.unlink() # Remove the test file - except PermissionError: - logger.error(f"Permission denied: Cannot write to directory {logo_dir}") - return False - except Exception as e: - logger.error(f"Directory access error for {logo_dir}: {e}") - return False - + # Create a simple placeholder logo logo = Image.new('RGBA', (64, 64), (100, 100, 100, 255)) # Gray background draw = ImageDraw.Draw(logo) @@ -774,14 +867,16 @@ class LogoDownloader: # proof the logo was fetched. metadata = PngInfo() metadata.add_text(PLACEHOLDER_MARKER, str(time.time())) - logo.save(filepath, "PNG", pnginfo=metadata) - - # Set proper file permissions after saving - ensure_file_permissions(filepath, get_assets_file_mode()) + # Atomic, and it sets the asset mode; an unwritable directory + # surfaces here as PermissionError. + save_png_atomically(logo, filepath, pnginfo=metadata) logger.info(f"Created placeholder logo for {team_abbreviation} at {filepath}") return True - + + except PermissionError as e: + logger.error(f"Permission denied: Cannot write placeholder logo to {logo_dir}: {e}") + return False except Exception as e: logger.error(f"Failed to create placeholder logo for {team_abbreviation}: {e}") return False @@ -837,6 +932,31 @@ def get_soccer_league_key(league_code: str) -> str: return f"soccer_{league_code}" +_thread_state = threading.local() + + +def shared_downloader() -> LogoDownloader: + """The calling thread's reusable LogoDownloader. + + ``download_missing_logo`` used to build a new downloader -- a new + ``requests.Session``, retry adapter and connection pool -- for every logo. + Reusing one keeps connections to ESPN's CDN alive between logos. + + One per thread rather than one behind a lock: ``requests.Session`` is not + documented as safe for concurrent use, and plugins download on worker + threads (football-scoreboard runs a small pool precisely so downloads + overlap). A lock would serialise every plugin's downloads behind the + slowest one -- up to 30s per attempt, with retries. Pool threads persist, + so each still reuses its own session; a thread's downloader goes with the + thread. + """ + downloader = getattr(_thread_state, 'downloader', None) + if downloader is None: + downloader = LogoDownloader() + _thread_state.downloader = downloader + return downloader + + # Convenience function for easy integration def download_missing_logo(league: str, team_id: str, team_abbreviation: str, logo_path: Path, logo_url: str | None = None, create_placeholder: bool = True) -> bool: """ @@ -852,8 +972,8 @@ def download_missing_logo(league: str, team_id: str, team_abbreviation: str, log Returns: True if logo exists or was successfully downloaded, False otherwise """ - downloader = LogoDownloader() - + downloader = shared_downloader() + # Use the directory from the logo_path parameter (respects config settings) logo_path = Path(logo_path) if not logo_path.is_absolute(): @@ -920,5 +1040,5 @@ def download_all_logos_for_league(league: str, force_download: bool = False) -> Returns: Tuple of (downloaded_count, failed_count) """ - downloader = LogoDownloader() + downloader = shared_downloader() return downloader.download_missing_logos_for_league(league, force_download) diff --git a/test/test_http_headers.py b/test/test_http_headers.py new file mode 100644 index 00000000..bc5836a1 --- /dev/null +++ b/test/test_http_headers.py @@ -0,0 +1,80 @@ +"""One set of HTTP headers for core's ESPN/data requests. + +The logo downloader and the background data service each carried their own +header dict with a placeholder User-Agent (``yourusername/LEDMatrix; +contact@example.com``) -- the kind of nonconforming token ESPN began 403ing +around 2026-08-04 -- and a hand-set ``Accept-Encoding: ... br`` although brotli +is not installed, so a ``br`` body could not have been decoded. +""" + +from unittest.mock import MagicMock + +import pytest + +from src.common.api_helper import DEFAULT_HTTP_HEADERS, USER_AGENT, APIHelper + + +def _lower_keys(headers): + return {k.lower(): v for k, v in headers.items()} + + +class TestSharedHeaders: + def test_user_agent_names_the_project(self): + assert USER_AGENT == 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)' + assert DEFAULT_HTTP_HEADERS['User-Agent'] == USER_AGENT + + def test_no_hand_set_accept_encoding(self): + assert 'accept-encoding' not in _lower_keys(DEFAULT_HTTP_HEADERS) + + def test_is_read_only(self): + with pytest.raises(TypeError): + DEFAULT_HTTP_HEADERS['User-Agent'] = 'x' # type: ignore[index] + + def test_api_helper_sends_the_same_user_agent(self): + assert APIHelper().session.headers['User-Agent'] == USER_AGENT + + def test_api_helper_does_not_hand_set_brotli(self): + assert 'br' not in APIHelper().session.headers.get('Accept-Encoding', '') + + def test_logo_helper_sends_the_same_user_agent(self): + from src.common.logo_helper import LogoHelper + assert LogoHelper(display_width=64, display_height=32).session.headers['User-Agent'] == USER_AGENT + + +class TestLogoDownloaderHeaders: + def test_uses_the_shared_headers(self): + from src.logo_downloader import LogoDownloader + headers = LogoDownloader().headers + assert headers['User-Agent'] == USER_AGENT + assert 'accept-encoding' not in _lower_keys(headers) + assert 'yourusername' not in str(headers) + + def test_instance_headers_are_a_private_copy(self): + from src.logo_downloader import LogoDownloader + downloader = LogoDownloader() + downloader.headers['X-Test'] = '1' + assert 'X-Test' not in DEFAULT_HTTP_HEADERS + assert 'X-Test' not in LogoDownloader().headers + + def test_logo_request_sends_the_user_agent_and_asks_for_an_image(self, tmp_path): + from src.logo_downloader import LogoDownloader + downloader = LogoDownloader() + downloader.session.get = MagicMock(side_effect=RuntimeError("stop")) + downloader.download_logo("http://x/a.png", tmp_path / "A.png", "A") + sent = _lower_keys(downloader.session.get.call_args.kwargs['headers']) + assert sent['user-agent'] == USER_AGENT + assert sent['accept'].startswith('image/') + assert 'accept-encoding' not in sent + + +class TestBackgroundDataServiceHeaders: + def test_uses_the_shared_headers(self): + from src.background_data_service import BackgroundDataService + service = BackgroundDataService(MagicMock(), max_workers=1) + try: + headers = service.default_headers + assert headers['User-Agent'] == USER_AGENT + assert 'accept-encoding' not in _lower_keys(headers) + assert 'yourusername' not in str(headers) + finally: + service.shutdown(wait=False) diff --git a/test/test_logo_downloader.py b/test/test_logo_downloader.py index c14b3391..a303b087 100644 --- a/test/test_logo_downloader.py +++ b/test/test_logo_downloader.py @@ -7,16 +7,20 @@ ensure_logo_directory, and the download_missing_logo function path (with HTTP mocked). """ +import io import os +import threading import time import pytest +import requests from pathlib import Path from unittest.mock import patch, Mock, MagicMock from PIL import Image from PIL.PngImagePlugin import PngInfo +import src.logo_downloader as logo_downloader_module from src.logo_downloader import ( PLACEHOLDER_BG, PLACEHOLDER_MARKER, @@ -351,3 +355,266 @@ class TestRefreshPlaceholderTimestamp: def test_missing_file_is_not_an_error(self, tmp_path): assert refresh_placeholder_timestamp(tmp_path / "nope.png") is False + + +# --------------------------------------------------------------------------- +# download_logo: the download the scoreboard plugins actually use +# +# It used to read response.content with no size cap, write straight to the +# final path (so a failed or corrupt download could be left there and then be +# cached as the logo), and build a fresh Session for every logo. It now goes +# through fetch_logo, the hardened download LogoHelper also uses. +# --------------------------------------------------------------------------- + +def _png(img: Image.Image) -> bytes: + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + + +def _stream(body: bytes, content_type: str = "image/png", chunk: int = 1024, + die_after: int | None = None): + """A streamed requests.Response stand-in. + + ``die_after`` makes the transfer fail after that many bytes, the way a + reset connection does mid-body. + """ + response = MagicMock() + response.__enter__.return_value = response + response.__exit__.return_value = False + response.raise_for_status = MagicMock() + response.headers = {"content-type": content_type} + + def _iter_content(*_args, **_kwargs): + sent = 0 + for i in range(0, len(body), chunk): + if die_after is not None and sent >= die_after: + raise requests.exceptions.ChunkedEncodingError("connection reset") + piece = body[i:i + chunk] + sent += len(piece) + yield piece + + response.iter_content = _iter_content + return response + + +@pytest.fixture +def fresh_thread_state(monkeypatch): + """Each test gets its own per-thread downloader cache.""" + monkeypatch.setattr(logo_downloader_module, "_thread_state", threading.local()) + + +@pytest.fixture +def downloader(): + return LogoDownloader() + + +@pytest.fixture +def old_logo(tmp_path): + """A logo already on disk that a failed re-download must not damage.""" + path = tmp_path / "PHI.png" + Image.new("RGBA", (30, 30), (1, 2, 3, 255)).save(path) + return path, path.read_bytes() + + +def _leftovers(directory: Path, keep: str | None = None): + return sorted(p.name for p in directory.iterdir() if p.name != keep) + + +class TestDownloadLogoHardening: + def test_valid_logo_is_saved_as_rgba_png(self, downloader, tmp_path): + target = tmp_path / "PHI.png" + downloader.session.get = MagicMock( + return_value=_stream(_png(Image.new("RGB", (20, 10), (9, 8, 7))))) + assert downloader.download_logo("http://x/phi.png", target, "PHI") is True + with Image.open(target) as img: + assert img.format == "PNG" and img.mode == "RGBA" + assert img.getpixel((0, 0)) == (9, 8, 7, 255) + assert _leftovers(tmp_path, keep="PHI.png") == [] + # Streamed, so the size cap applies before the body is buffered. + assert downloader.session.get.call_args.kwargs["stream"] is True + + def test_oversized_response_is_rejected_and_leaves_no_file( + self, downloader, tmp_path, monkeypatch): + monkeypatch.setattr(logo_downloader_module, "MAX_LOGO_BYTES", 4096) + target = tmp_path / "BIG.png" + body = _png(Image.new("RGB", (8, 8))) + b"\x00" * 8192 + downloader.session.get = MagicMock(return_value=_stream(body)) + assert downloader.download_logo("http://x/big.png", target, "BIG") is False + assert list(tmp_path.iterdir()) == [] + + def test_oversized_response_does_not_replace_an_existing_logo( + self, downloader, old_logo, monkeypatch): + path, before = old_logo + monkeypatch.setattr(logo_downloader_module, "MAX_LOGO_BYTES", 4096) + downloader.session.get = MagicMock( + return_value=_stream(b"\x89PNG" + b"\x00" * 8192)) + assert downloader.download_logo("http://x/big.png", path, "PHI") is False + assert path.read_bytes() == before + assert _leftovers(path.parent, keep=path.name) == [] + + def test_mid_download_failure_leaves_no_partial_file(self, downloader, tmp_path): + target = tmp_path / "CUT.png" + body = _png(Image.new("RGB", (200, 200), (5, 5, 5))) + b"\x00" * 4096 + downloader.session.get = MagicMock( + return_value=_stream(body, chunk=256, die_after=512)) + assert downloader.download_logo("http://x/cut.png", target, "CUT") is False + assert list(tmp_path.iterdir()) == [] + + def test_mid_download_failure_keeps_the_previous_logo(self, downloader, old_logo): + # The old code opened the final path for writing before the body + # arrived, so a dropped connection truncated the logo it was replacing. + path, before = old_logo + body = _png(Image.new("RGB", (200, 200), (5, 5, 5))) + b"\x00" * 4096 + downloader.session.get = MagicMock( + return_value=_stream(body, chunk=256, die_after=512)) + assert downloader.download_logo("http://x/cut.png", path, "PHI") is False + assert path.read_bytes() == before + assert _leftovers(path.parent, keep=path.name) == [] + + def test_non_image_content_type_is_rejected(self, downloader, old_logo): + # Rejected on the label alone, before the body is trusted: these bytes + # would decode, so only the content-type check stops them. + path, before = old_logo + body = _png(Image.new("RGB", (8, 8), (250, 0, 0))) + downloader.session.get = MagicMock( + return_value=_stream(body, content_type="text/html")) + assert downloader.download_logo("http://x/404", path, "PHI") is False + assert path.read_bytes() == before + assert _leftovers(path.parent, keep=path.name) == [] + + def test_bytes_that_do_not_decode_are_rejected(self, downloader, old_logo): + # A server that labels an error page image/png must not get it cached. + path, before = old_logo + downloader.session.get = MagicMock( + return_value=_stream(b"oops", content_type="image/png")) + assert downloader.download_logo("http://x/lie.png", path, "PHI") is False + assert path.read_bytes() == before + assert _leftovers(path.parent, keep=path.name) == [] + + def test_http_error_keeps_the_previous_logo(self, downloader, old_logo): + path, before = old_logo + response = _stream(b"") + response.raise_for_status.side_effect = requests.exceptions.HTTPError("503") + downloader.session.get = MagicMock(return_value=response) + assert downloader.download_logo("http://x/phi.png", path, "PHI") is False + assert path.read_bytes() == before + assert _leftovers(path.parent, keep=path.name) == [] + + def test_unwritable_directory_returns_false(self, downloader, tmp_path): + downloader.session.get = MagicMock() + with patch("src.logo_downloader.tempfile.mkstemp", + side_effect=PermissionError("read-only")): + assert downloader.download_logo( + "http://x/phi.png", tmp_path / "PHI.png", "PHI") is False + downloader.session.get.assert_not_called() + + +class TestDownloadLogoTransparency: + """Plugins paste logos with the image as its own mask; alpha must survive.""" + + def _download(self, downloader, tmp_path, body, content_type="image/png"): + target = tmp_path / "LOGO.png" + downloader.session.get = MagicMock(return_value=_stream(body, content_type)) + assert downloader.download_logo("http://x/logo", target, "LOGO") is True + with Image.open(target) as img: + img.load() + return img.copy(), img.format + + def test_rgba_alpha_is_kept_exactly(self, downloader, tmp_path): + src = Image.new("RGBA", (16, 16), (0, 0, 0, 0)) + for x in range(16): + src.putpixel((x, 3), (200, 100, 50, x * 16)) + out, _ = self._download(downloader, tmp_path, _png(src)) + assert out.mode == "RGBA" + assert list(out.getdata()) == list(src.getdata()) + + def test_palette_transparency_becomes_alpha(self, downloader, tmp_path): + src = Image.new("P", (8, 8), 0) + src.putpalette([0, 0, 0, 255, 0, 0] + [0] * (254 * 3)) + src.putpixel((4, 4), 1) + buf = io.BytesIO() + src.save(buf, "PNG", transparency=0) + out, _ = self._download(downloader, tmp_path, buf.getvalue()) + assert out.mode == "RGBA" + assert out.getpixel((0, 0))[3] == 0 + assert out.getpixel((4, 4)) == (255, 0, 0, 255) + + def test_greyscale_transparency_becomes_alpha(self, downloader, tmp_path): + src = Image.new("L", (8, 8), 0) + src.putpixel((2, 2), 255) + buf = io.BytesIO() + src.save(buf, "PNG", transparency=0) + out, _ = self._download(downloader, tmp_path, buf.getvalue()) + assert out.getpixel((0, 0))[3] == 0 + assert out.getpixel((2, 2)) == (255, 255, 255, 255) + + def test_jpeg_is_stored_as_opaque_rgba_png(self, downloader, tmp_path): + buf = io.BytesIO() + Image.new("RGB", (8, 8), (10, 200, 30)).save(buf, "JPEG", quality=95) + out, fmt = self._download(downloader, tmp_path, buf.getvalue(), "image/jpeg") + assert fmt == "PNG" and out.mode == "RGBA" + assert out.getchannel("A").getextrema() == (255, 255) + + +class TestSharedDownloader: + def test_download_missing_logo_reuses_one_session(self, tmp_path, fresh_thread_state): + sessions = [] + real_session = requests.Session + + def counting_session(*args, **kwargs): + s = real_session(*args, **kwargs) + sessions.append(s) + return s + + with patch("src.logo_downloader.requests.Session", side_effect=counting_session): + with patch.object(LogoDownloader, "download_logo", return_value=True) as dl: + for abbr in ("AAA", "BBB", "CCC"): + assert download_missing_logo( + "nfl", "1", abbr, tmp_path / f"{abbr}.png", + logo_url=f"http://x/{abbr}.png", + create_placeholder=False) is True + assert dl.call_count == 3 + assert len(sessions) == 1 + + def test_each_thread_gets_its_own_downloader(self, fresh_thread_state): + here = logo_downloader_module.shared_downloader() + assert logo_downloader_module.shared_downloader() is here + seen = [] + t = threading.Thread(target=lambda: seen.append( + logo_downloader_module.shared_downloader())) + t.start() + t.join() + assert seen and seen[0] is not here + assert seen[0].session is not here.session + + +class TestPlaceholderWrite: + def test_no_write_probe_file_is_created(self, tmp_path): + created = [] + real_touch = Path.touch + + def spy_touch(self, *args, **kwargs): + created.append(self.name) + return real_touch(self, *args, **kwargs) + + with patch.object(Path, "touch", spy_touch): + assert LogoDownloader().create_placeholder_logo("COLL", str(tmp_path)) is True + assert "test_write.tmp" not in created + assert sorted(p.name for p in tmp_path.iterdir()) == ["COLL.png"] + + def test_unwritable_directory_returns_false(self, tmp_path): + with patch.object(LogoDownloader, "ensure_logo_directory", return_value=True), \ + patch("src.logo_downloader.tempfile.mkstemp", + side_effect=PermissionError("read-only")): + assert LogoDownloader().create_placeholder_logo("COLL", str(tmp_path)) is False + assert list(tmp_path.iterdir()) == [] + + def test_failed_save_keeps_the_previous_file(self, tmp_path): + path = tmp_path / "COLL.png" + Image.new("RGBA", (30, 30), (1, 2, 3, 255)).save(path) + before = path.read_bytes() + with patch("src.logo_downloader.os.replace", side_effect=OSError("disk full")): + assert LogoDownloader().create_placeholder_logo("COLL", str(tmp_path)) is False + assert path.read_bytes() == before + assert sorted(p.name for p in tmp_path.iterdir()) == ["COLL.png"] diff --git a/test/test_logo_helper.py b/test/test_logo_helper.py index 0ae7ec44..9ce4eeaf 100644 --- a/test/test_logo_helper.py +++ b/test/test_logo_helper.py @@ -33,7 +33,7 @@ def _no_real_chmod(monkeypatch): # Keep the permission helpers out of the way: their own env detection # is not what these tests are about. monkeypatch.setattr("src.common.logo_helper.ensure_directory_permissions", MagicMock()) - monkeypatch.setattr("src.common.logo_helper.ensure_file_permissions", MagicMock()) + monkeypatch.setattr("src.logo_downloader.ensure_file_permissions", MagicMock()) @pytest.fixture @@ -48,7 +48,8 @@ def write_logo(path: Path, size=(20, 20), color=(255, 0, 0), fmt="PNG") -> Path: return path -def fake_response(content: bytes, chunk_size: int = 64 * 1024): +def fake_response(content: bytes, chunk_size: int = 64 * 1024, + content_type: str = "image/png"): """Stand-in for a streamed requests.Response. _download_logo opens `with session.get(..., stream=True)` and reads @@ -61,6 +62,7 @@ def fake_response(content: bytes, chunk_size: int = 64 * 1024): response.__enter__.return_value = response response.__exit__.return_value = False response.raise_for_status = MagicMock() + response.headers = {"content-type": content_type} def _iter_content(*_args, **_kwargs): for i in range(0, len(content), chunk_size): @@ -80,6 +82,7 @@ def endless_response(chunk: bytes = b"\x00" * 65536): response.__enter__.return_value = response response.__exit__.return_value = False response.raise_for_status = MagicMock() + response.headers = {"content-type": "image/png"} def _iter_content(*_args, **_kwargs): while True: @@ -208,7 +211,7 @@ class TestLoadLogoWithDownload: # stream=True is load-bearing: it is what lets the size cap apply # before the body is buffered. helper.session.get.assert_called_once_with( - "http://x/logo.png", timeout=30, stream=True) + "http://x/logo.png", headers=None, timeout=30, stream=True) def test_download_failure_falls_back_to_placeholder(self, helper, tmp_path): helper.session.get = MagicMock( @@ -241,7 +244,7 @@ class TestDownloadLogo: path.parent.mkdir() helper.session.get = MagicMock(return_value=fake_response(png_bytes())) with patch("src.common.logo_helper.ensure_directory_permissions") as dirs, \ - patch("src.common.logo_helper.ensure_file_permissions") as files: + patch("src.logo_downloader.ensure_file_permissions") as files: helper._download_logo("http://x/logo.png", path) assert path.exists() dirs.assert_called_once() @@ -284,6 +287,7 @@ class TestDownloadLogo: response.__enter__.return_value = response response.__exit__.return_value = False response.raise_for_status = MagicMock() + response.headers = {"content-type": "image/png"} response.iter_content = _dies_midway helper.session.get = MagicMock(return_value=response) @@ -305,7 +309,7 @@ class TestDownloadLogo: seen.append(name) return fd, name - with patch("src.common.logo_helper.tempfile.mkstemp", side_effect=record): + with patch("src.logo_downloader.tempfile.mkstemp", side_effect=record): helper.session.get = MagicMock(return_value=fake_response(png_bytes())) helper._download_logo("http://x/logo.png", path) helper.session.get = MagicMock(return_value=fake_response(png_bytes())) @@ -351,7 +355,7 @@ class TestDownloadLogo: def load(self): raise Image.DecompressionBombError("too many pixels") - monkeypatch.setattr("src.common.logo_helper.Image.open", lambda *a, **kw: Bomb()) + monkeypatch.setattr("src.logo_downloader.Image.open", lambda *a, **kw: Bomb()) with pytest.raises(Image.DecompressionBombError): helper._download_logo("http://x/bomb.png", path) assert not path.exists() @@ -420,7 +424,8 @@ class TestPlaceholderLogo: class TestSessionConfiguration: def test_user_agent_and_accept_headers(self, helper): - assert helper.session.headers["User-Agent"] == "LEDMatrix-Common/1.0" + from src.common.api_helper import USER_AGENT + assert helper.session.headers["User-Agent"] == USER_AGENT assert helper.session.headers["Accept"] == "image/*"