feat(fetch): shared fetch service, stage 1 (pooling, merging, host budgets, counters) (#702)

Core's own HTTP fetch paths (APIHelper, fetch_espn_scoreboard and its date chunks, BackgroundDataService, BaseOddsManager.get_odds) go through one service in src/common/fetch_service.py: shared connection pools per retry policy, merged identical in-flight GETs, per-host token-bucket budgets (fetch_service.rate_limits), and per-plugin request counters published to GET /api/v3/plugins/fetch-stats. Return values, exceptions, cache keys, TTLs and retry policies are unchanged. Core-internal in this release; plugins should not import it directly yet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chuck
2026-10-01 15:38:11 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9edeb6da14
commit 4be53d048b
19 changed files with 2407 additions and 31 deletions
+5 -1
View File
@@ -10,6 +10,7 @@ from typing import Any, Dict, Optional, Callable
from threading import Thread
import logging
from src.common.fetch_service import plugin_scope
from src.exceptions import PluginError
from src.logging_config import get_logger
from src.error_aggregator import record_error
@@ -83,7 +84,10 @@ class PluginExecutor:
def target():
try:
result_container['value'] = operation()
# Fetches made by the operation (and by threads the core
# starts from it) are counted against this plugin.
with plugin_scope(plugin_id):
result_container['value'] = operation()
result_container['completed'] = True
except Exception as e:
result_container['exception'] = e
+22 -13
View File
@@ -32,6 +32,7 @@ from src.plugin_system.schema_manager import (
from src.plugin_system.plugin_dirs import (
ManifestStatus, PluginDirectoryIndex, resolve_plugin_dir,
)
from src.common.fetch_service import plugin_scope, register_plugin_directory
from src.common.permission_utils import (
ensure_directory_permissions,
get_plugin_dir_mode
@@ -423,6 +424,11 @@ class PluginManager:
# Update mapping if found via search
if plugin_id not in self.plugin_directories:
self.plugin_directories[plugin_id] = plugin_dir
# Code under this directory is this plugin's: the fetch service
# counts a request against it even from a thread the plugin
# started itself (src/common/fetch_service.py, caller identity).
register_plugin_directory(plugin_id, plugin_dir)
# Get plugin config
if self.config_manager:
@@ -462,18 +468,20 @@ class PluginManager:
config = dict(config)
config['enabled'] = True
# Use PluginLoader to load plugin
plugin_instance, _module = self.plugin_loader.load_plugin(
plugin_id=plugin_id,
manifest=manifest,
plugin_dir=plugin_dir,
config=config,
display_manager=self.display_manager,
cache_manager=self.cache_manager,
plugin_manager=self,
install_deps=True,
plugins_dir=self.plugins_dir,
)
# Use PluginLoader to load plugin. Fetches the constructor makes
# count against the plugin.
with plugin_scope(plugin_id):
plugin_instance, _module = self.plugin_loader.load_plugin(
plugin_id=plugin_id,
manifest=manifest,
plugin_dir=plugin_dir,
config=config,
display_manager=self.display_manager,
cache_manager=self.cache_manager,
plugin_manager=self,
install_deps=True,
plugins_dir=self.plugins_dir,
)
# Register plugin-shipped fonts with the FontManager (if any).
# Plugin manifests can declare a "fonts" block that ships custom
@@ -527,7 +535,8 @@ class PluginManager:
# Call on_enable if plugin is enabled
if hasattr(plugin_instance, 'on_enable'):
try:
plugin_instance.on_enable()
with plugin_scope(plugin_id):
plugin_instance.on_enable()
except Exception:
# Undo the registration above before the outer
# handler marks it ERROR: left in self.plugins, the