feat(fetch): shared fetch service, stage 1 (pooling, merging, host budgets, counters) (#702)

Core's own HTTP fetch paths (APIHelper, fetch_espn_scoreboard and its date chunks, BackgroundDataService, BaseOddsManager.get_odds) go through one service in src/common/fetch_service.py: shared connection pools per retry policy, merged identical in-flight GETs, per-host token-bucket budgets (fetch_service.rate_limits), and per-plugin request counters published to GET /api/v3/plugins/fetch-stats. Return values, exceptions, cache keys, TTLs and retry policies are unchanged. Core-internal in this release; plugins should not import it directly yet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/background_data_service.py

@@ -27,6 +27,13 @@ from concurrent.futures import ThreadPoolExecutor
 import pytz
 from src.cache_manager import CacheManager
 from src.common.json_body import response_json
+from src.common.fetch_service import (
+    current_plugin_id,
+    fetch_get,
+    get_fetch_service,
+    plugin_scope,
+    share_connection_pool,
+)
 from src.common.espn_dates import (
     RANGE_RETRY_SECONDS,
     _note_range_rejected,
@@ -78,6 +85,9 @@ class FetchRequest:
     commit_claimed: bool = False
     result: Optional[Any] = None
     error: Optional[str] = None
+    # The plugin that submitted the request, so the fetch service counts the
+    # worker's requests against it (fetch_service, caller identity).
+    owner: Optional[str] = None
 
 @dataclass
 class FetchResult:
@@ -119,6 +129,12 @@ class _ConnectionRetryingSession:
     def __init__(self, session):
         self._session = session
 
+    @property
+    def fetch_identity_session(self):
+        """The wrapped Session, whose headers and adapter the fetch service
+        reads to key this request (src/common/fetch_service.py)."""
+        return self._session
+
     def get(self, *args, **kwargs):
         for attempt in range(self.ATTEMPTS):
             try:
@@ -196,9 +212,12 @@ class BackgroundDataService:
         # connection errors three times, a dead network cost up to 16
         # connection attempts per request and held one of the few worker
         # threads for all of them.
+        #
+        # The adapter is the fetch service's shared no-retry one: the same
+        # max_retries=0, with the connection pool shared with the other core
+        # sessions that do not retry (the odds managers).
         self.session = requests.Session()
-        self.session.mount('http://', requests.adapters.HTTPAdapter(max_retries=0))
-        self.session.mount('https://', requests.adapters.HTTPAdapter(max_retries=0))
+        share_connection_pool(self.session, max_retries=0)
         
         # Default headers: core's shared set (real User-Agent, no hand-set
         # Accept-Encoding) -- see src/common/api_helper.py.
@@ -299,6 +318,10 @@ class BackgroundDataService:
         if url.split('?', 1)[0].rstrip('/').endswith('/scoreboard'):
             params = clamp_espn_limit(params)
 
+        # Who asked, resolved on the submitting thread: the worker thread
+        # runs no plugin code, so it could not tell (fetch_service).
+        owner = current_plugin_id()
+
         # Create fetch request
         request = FetchRequest(
             id=request_id,
@@ -311,7 +334,8 @@ class BackgroundDataService:
             timeout=timeout or self.request_timeout,
             max_retries=max_retries,
             priority=priority,
-            callback=callback
+            callback=callback,
+            owner=owner,
         )
         
         with self._lock:
@@ -330,6 +354,7 @@ class BackgroundDataService:
                 self.stats['deduplicated_requests'] = (
                     self.stats.get('deduplicated_requests', 0) + 1
                 )
+                get_fetch_service().note_merged(url, owner)
                 logger.info(
                     "Joined in-flight fetch %s for %s (cache_key=%s) instead of "
                     "starting a duplicate", existing_id, sport, cache_key
@@ -357,6 +382,11 @@ class BackgroundDataService:
         Returns:
             Fetch result with data or error information
         """
+        with plugin_scope(request.owner):
+            return self._fetch_data_worker_scoped(request)
+
+    def _fetch_data_worker_scoped(self, request: FetchRequest) -> FetchResult:
+        """_fetch_data_worker's body, run with the submitter as the caller."""
         start_time = time.time()
         result = FetchResult(request_id=request.id, success=False, retry_count=request.retry_count)
         
@@ -621,8 +651,14 @@ class BackgroundDataService:
         
         for attempt in range(request.max_retries + 1):
             try:
-                response = self.session.get(
+                # Not shared with an identical request in flight: this
+                # service cancels and replaces fetches, and a replacement
+                # must not join the one it replaced. Its own cache_key
+                # dedup already merges what should be merged.
+                response = fetch_get(
+                    self.session,
                     request.url,
+                    share_in_flight=False,
                     params=request.params,
                     headers=request.headers,
                     timeout=request.timeout

src/base_odds_manager.py

@@ -19,6 +19,7 @@ import json
 from typing import Dict, Any, Optional, List, cast
 
 from src.common.api_helper import DEFAULT_HTTP_HEADERS
+from src.common.fetch_service import fetch_get, share_connection_pool
 
 
 
@@ -59,7 +60,13 @@ class BaseOddsManager:
         # Deliberately no retry adapter, unlike api_helper: retries multiply
         # request_timeout, which is set to 5s precisely to stay inside that
         # budget. One try, then the cooldown below.
+        #
+        # Every scoreboard league manager builds one of these, so the session
+        # mounts the fetch service's shared no-retry adapter: the same single
+        # try, over one connection pool per host for all of them instead of
+        # one pool per instance.
         self.session = requests.Session()
+        share_connection_pool(self.session, max_retries=0)
         self.session.headers.update(DEFAULT_HTTP_HEADERS)
         
         # Configuration with defaults
@@ -168,7 +175,7 @@ class BaseOddsManager:
             url = f"{self.base_url}/{sport}/leagues/{espn_league}/events/{event_id}/competitions/{event_id}/odds"
             self.logger.debug(f"Requesting odds from URL: {url}")
             
-            response = self.session.get(url, timeout=self.request_timeout)
+            response = fetch_get(self.session, url, timeout=self.request_timeout)
             response.raise_for_status()
             raw_data = response.json()
 

src/common/README.md

@@ -27,6 +27,7 @@ Rules for the package:
 | [`bdf_font`](#bdf_font) | Load and draw BDF bitmap fonts | Yes, if drawing BDF text directly | 3.5.0 |
 | [`espn_dates`](#espn_dates) | Fetch ESPN scoreboards across a date range | Yes (scoreboards) | 3.5.0 |
 | [`favorite_team_check`](#favorite_team_check) | Log why a favourite team code shows nothing | Yes (scoreboards) | 3.6.0 |
+| [`fetch_service`](#fetch_service) | Pooled, merged, budgeted and counted HTTP for core fetch paths | No, core-internal (reached through `api_helper` and `espn_dates`) | n/a |
 | [`font_layout`](#font_layout) | Reproducible TrueType loading, crisp sizes | Yes | 3.4.0 |
 | [`frame_timing`](#frame_timing) | Timing of every presented frame, stall watchdog | No, core-internal | n/a |
 | [`json_body`](#json_body) | Parse a response body as JSON, with orjson if installed | Optional (large payloads) | 3.5.0 |
@@ -108,7 +109,9 @@ and truncates results when `limit` is above 500. `fetch_espn_scoreboard()`
 splits a range into month and day requests ESPN accepts and merges the
 results; `espn_date_chunks()`, `fetch_espn_date_chunks()`,
 `clamp_espn_limit()` and `merge_scoreboard_payloads()` are the pieces.
-Scoreboard plugins also bundle a copy for older cores.
+Every request goes through [`fetch_service`](#fetch_service), the chunks
+counted against the plugin that asked. Scoreboard plugins also bundle a copy
+for older cores.
 
 ### favorite_team_check
 
@@ -121,6 +124,23 @@ says the league has nothing on yet; `reset()` re-arms it after a config edit.
 Diagnostics only: every failure is swallowed. Scoreboard plugins also bundle
 a copy for older cores.
 
+### fetch_service
+
+[`fetch_service.py`](fetch_service.py). Core-internal for now. Every core
+fetch path -- `APIHelper.get`/`post`, `espn_dates` (so every scoreboard's
+ESPN scoreboard fetch and `SportsFetchMixin`), `BackgroundDataService` and
+`BaseOddsManager` -- calls `fetch_get(session, url, ...)` instead of
+`session.get(url, ...)`. Same arguments, return value and exceptions; on top
+it shares one connection pool per host per retry policy
+(`share_connection_pool`), merges identical GETs in flight, applies per-host
+token buckets (`fetch_service.rate_limits` in config.json; ESPN gets 20/s,
+burst 200), revalidates with server-sent `ETag`/`Last-Modified` and counts
+requests per plugin and per host. The display publishes the counters
+(`FetchStatsPublisher`) for `GET /api/v3/plugins/fetch-stats`. Which plugin
+made a request comes from `plugin_scope()`, set by the plugin executor, or
+else from the plugin directory on the stack. See
+[docs/PLUGIN_API_REFERENCE.md](../../docs/PLUGIN_API_REFERENCE.md#fetching-data).
+
 ### font_layout
 
 [`font_layout.py`](font_layout.py). `load_truetype(path, size)` is

src/common/api_helper.py

@@ -11,10 +11,10 @@ import time
 from datetime import datetime
 from types import MappingProxyType
 from src.common.espn_dates import ESPN_MAX_LIMIT
+from src.common.fetch_service import fetch_get, fetch_post, share_connection_pool
 from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, cast
 
 import requests
-from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
 if TYPE_CHECKING:
@@ -45,7 +45,11 @@ class APIHelper:
 
     - Requests go through one ``requests.Session`` that retries GET, HEAD
       and OPTIONS on 429 and 5xx with exponential backoff, and sends
-      :data:`DEFAULT_HTTP_HEADERS`.
+      :data:`DEFAULT_HTTP_HEADERS`. Its connection pool is shared with every
+      other helper using the same retry policy, and requests go through the
+      core fetch service (``src/common/fetch_service.py``): identical GETs in
+      flight are merged, hosts with a budget are paced, and requests are
+      counted per plugin. Return values and errors are unchanged.
     - Consecutive requests from one helper are spaced at least
       ``set_rate_limit()`` seconds apart (1 second by default). A cache hit
       does not count.
@@ -81,9 +85,10 @@ class APIHelper:
             status_forcelist=[429, 500, 502, 503, 504],
             allowed_methods=["GET", "HEAD", "OPTIONS"]
         )
-        adapter = HTTPAdapter(max_retries=retry_strategy)
-        self.session.mount("https://", adapter)
-        self.session.mount("http://", adapter)
+        # The shared adapter for this retry policy: the same retries as a
+        # private HTTPAdapter(max_retries=retry_strategy), with the connection
+        # pool shared by every helper (fetch_service).
+        share_connection_pool(self.session, retry_strategy)
         
         self.session.headers.update({**DEFAULT_HTTP_HEADERS, 'Connection': 'keep-alive'})
         
@@ -128,7 +133,8 @@ class APIHelper:
                 request_headers.update(headers)
             
             # Make request
-            response = self.session.get(
+            response = fetch_get(
+                self.session,
                 url, 
                 params=params,
                 headers=request_headers,
@@ -255,7 +261,8 @@ class APIHelper:
             if headers:
                 request_headers.update(headers)
             
-            response = self.session.post(
+            response = fetch_post(
+                self.session,
                 url,
                 data=data,
                 json=json_data,

src/common/espn_dates.py

@@ -32,6 +32,7 @@ scoreboards ask every 30 seconds. After that the range is tried again, so the
 workaround retires itself if ESPN reverts.
 """
 
+import contextvars
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor
@@ -47,6 +48,21 @@ except ImportError:
     def response_json(response: Any) -> Any:
         return response.json()
 
+try:
+    # The core fetch service: counts, per-host budget, merging of identical
+    # requests. Same call, same result and errors as ``session.get``.
+    from src.common.fetch_service import fetch_get, pinned_caller
+except ImportError:
+    # Bundled copies on cores without it call the session directly.
+    import contextlib
+
+    def fetch_get(session: Any, url: str, *, share_in_flight: bool = True,
+                  **kwargs: Any) -> Any:
+        return session.get(url, **kwargs)
+
+    def pinned_caller() -> Any:
+        return contextlib.nullcontext()
+
 # Above this, ESPN returns a truncated list instead of an error. See module
 # docstring: 500 is the largest value measured to return complete data.
 ESPN_MAX_LIMIT = 500
@@ -195,7 +211,8 @@ def _fetch_one_chunk(
     logged and swallowed here rather than raised to the gather below.
     """
     try:
-        response = session.get(
+        response = fetch_get(
+            session,
             url,
             params=dict(params, dates=chunk, limit=ESPN_MAX_LIMIT),
             headers=headers,
@@ -220,6 +237,10 @@ def _fetch_chunks(
     callers keep ``chunks`` order from the returned list -- but it does mean
     the session is shared across threads, which is why this only ever issues
     GETs and never touches session state.
+
+    Each chunk runs in a copy of the caller's context, with the caller pinned
+    into it, so the fetch service counts the chunks against the plugin that
+    asked for the range rather than against the core.
     """
     if not chunks:
         return []
@@ -229,10 +250,15 @@ def _fetch_chunks(
     if len(chunks) == 1:
         return [fetch(chunks[0])]
     workers = min(ESPN_CHUNK_WORKERS, len(chunks))
+    with pinned_caller():
+        # One copy per chunk: a Context cannot be entered by two threads.
+        contexts = [contextvars.copy_context() for _ in chunks]
     with ThreadPoolExecutor(
         max_workers=workers, thread_name_prefix="espn-chunk",
     ) as pool:
-        return list(pool.map(fetch, chunks))
+        futures = [pool.submit(context.run, fetch, chunk)
+                   for context, chunk in zip(contexts, chunks)]
+        return [future.result() for future in futures]
 
 
 def fetch_espn_date_chunks(
@@ -363,7 +389,7 @@ def fetch_espn_scoreboard(
         # real error to log, without spending the chunks a second time.
         chunks_tried = True
 
-    response = session.get(url, params=params, headers=headers, timeout=timeout)
+    response = fetch_get(session, url, params=params, headers=headers, timeout=timeout)
     if is_range and response.status_code == 400 and not chunks_tried:
         _note_range_rejected()
         if logger:

src/core_config_keys.py

@@ -29,6 +29,7 @@ CORE_CONFIG_KEYS = frozenset({
     'display',
     'sync',
     'plugin_system',
+    'fetch_service',
     # Older or optional core sections still found in existing config files.
     'logging',
     'network',

src/display_controller.py

@@ -132,6 +132,13 @@ class DisplayController:
         # The web interface's /api/v3/errors/* read what this publishes.
         from src.error_aggregator import start_error_snapshot_publisher
         start_error_snapshot_publisher(self.cache_manager)
+        # Host budgets and the other fetch_service settings, before any plugin
+        # fetches; the web UI's fetch statistics read what the publisher
+        # writes (src/common/fetch_service.py).
+        from src.common.fetch_service import (
+            configure_fetch_service, start_fetch_stats_publisher)
+        configure_fetch_service(self.config.get('fetch_service'))
+        self._fetch_stats_publisher = start_fetch_stats_publisher(self.cache_manager)
         logger.info("Config loaded in %.3f seconds (hot-reload: %s)", time.time() - start_time, enable_hot_reload)
         
         # Validate startup configuration. Errors are logged, not fatal. The
@@ -3819,6 +3826,9 @@ class DisplayController:
         read stale values after the user saves settings via the web UI.
         """
         self.config = new_config
+        # A no-op unless the fetch_service section itself changed.
+        from src.common.fetch_service import configure_fetch_service
+        configure_fetch_service(new_config.get('fetch_service'))
         self._normal_brightness = (
             self.config.get('display', {}).get('hardware', {}).get('brightness', 90)
         )
@@ -3886,6 +3896,11 @@ class DisplayController:
                 logger.warning("Error shutting down config service: %s", e)
         if getattr(self, '_font_usage_publisher', None) is not None:
             self._font_usage_publisher.stop()
+        if getattr(self, '_fetch_stats_publisher', None) is not None:
+            try:
+                self._fetch_stats_publisher.stop()
+            except Exception as e:
+                logger.warning("Error stopping the fetch statistics publisher: %s", e)
         # Publishes "stopped", so the web UI stops reporting what was loaded.
         if getattr(self, '_plugin_runtime_publisher', None) is not None:
             try:

src/plugin_system/plugin_executor.py

@@ -10,6 +10,7 @@ from typing import Any, Dict, Optional, Callable
 from threading import Thread
 import logging
 
+from src.common.fetch_service import plugin_scope
 from src.exceptions import PluginError
 from src.logging_config import get_logger
 from src.error_aggregator import record_error
@@ -83,7 +84,10 @@ class PluginExecutor:
         
         def target():
             try:
-                result_container['value'] = operation()
+                # Fetches made by the operation (and by threads the core
+                # starts from it) are counted against this plugin.
+                with plugin_scope(plugin_id):
+                    result_container['value'] = operation()
                 result_container['completed'] = True
             except Exception as e:
                 result_container['exception'] = e

src/plugin_system/plugin_manager.py

@@ -32,6 +32,7 @@ from src.plugin_system.schema_manager import (
 from src.plugin_system.plugin_dirs import (
     ManifestStatus, PluginDirectoryIndex, resolve_plugin_dir,
 )
+from src.common.fetch_service import plugin_scope, register_plugin_directory
 from src.common.permission_utils import (
     ensure_directory_permissions,
     get_plugin_dir_mode
@@ -423,6 +424,11 @@ class PluginManager:
             # Update mapping if found via search
             if plugin_id not in self.plugin_directories:
                 self.plugin_directories[plugin_id] = plugin_dir
+
+            # Code under this directory is this plugin's: the fetch service
+            # counts a request against it even from a thread the plugin
+            # started itself (src/common/fetch_service.py, caller identity).
+            register_plugin_directory(plugin_id, plugin_dir)
             
             # Get plugin config
             if self.config_manager:
@@ -462,18 +468,20 @@ class PluginManager:
                 config = dict(config)
                 config['enabled'] = True
             
-            # Use PluginLoader to load plugin
-            plugin_instance, _module = self.plugin_loader.load_plugin(
-                plugin_id=plugin_id,
-                manifest=manifest,
-                plugin_dir=plugin_dir,
-                config=config,
-                display_manager=self.display_manager,
-                cache_manager=self.cache_manager,
-                plugin_manager=self,
-                install_deps=True,
-                plugins_dir=self.plugins_dir,
-            )
+            # Use PluginLoader to load plugin. Fetches the constructor makes
+            # count against the plugin.
+            with plugin_scope(plugin_id):
+                plugin_instance, _module = self.plugin_loader.load_plugin(
+                    plugin_id=plugin_id,
+                    manifest=manifest,
+                    plugin_dir=plugin_dir,
+                    config=config,
+                    display_manager=self.display_manager,
+                    cache_manager=self.cache_manager,
+                    plugin_manager=self,
+                    install_deps=True,
+                    plugins_dir=self.plugins_dir,
+                )
             
             # Register plugin-shipped fonts with the FontManager (if any).
             # Plugin manifests can declare a "fonts" block that ships custom
@@ -527,7 +535,8 @@ class PluginManager:
                 # Call on_enable if plugin is enabled
                 if hasattr(plugin_instance, 'on_enable'):
                     try:
-                        plugin_instance.on_enable()
+                        with plugin_scope(plugin_id):
+                            plugin_instance.on_enable()
                     except Exception:
                         # Undo the registration above before the outer
                         # handler marks it ERROR: left in self.plugins, the