LEDMatrix/src/background_data_service.py

"""
Background Data Service for LEDMatrix

This service provides background threading capabilities for season data fetching
to prevent blocking the main display loop. It's designed to be used across
all sport managers for consistent background data management.

Key Features:
- Thread-safe data caching
- Automatic retry logic with exponential backoff
- Configurable timeouts and intervals
- Graceful error handling
- Progress tracking and logging
- Memory-efficient data storage
"""

import itertools
import time
from datetime import datetime
import logging
import threading
import requests
from typing import Dict, Any, Optional, Callable, List
from dataclasses import dataclass, field
from enum import Enum
from concurrent.futures import ThreadPoolExecutor
import pytz
from src.cache_manager import CacheManager
from src.common.json_body import response_json
from src.common.espn_dates import (
    RANGE_RETRY_SECONDS,
    _note_range_rejected,
    _ranges_known_rejected,
    clamp_espn_limit,
    fetch_espn_date_chunks,
    parse_espn_date_range,
)
# Configure logging
logger = logging.getLogger(__name__)

class FetchStatus(Enum):
    """Status of background fetch operations."""
    PENDING = "pending"
    IN_PROGRESS = "in_progress"
    COMPLETED = "completed"
    FAILED = "failed"
    CANCELLED = "cancelled"

@dataclass
class FetchRequest:
    """Represents a background fetch request."""
    id: str
    sport: str
    year: int
    cache_key: str
    url: str
    params: Dict[str, Any] = field(default_factory=dict)
    headers: Dict[str, str] = field(default_factory=dict)
    timeout: int = 30
    retry_count: int = 0
    max_retries: int = 3
    # Recorded but not acted on: requests go straight to the thread pool in
    # submission order. Kept because plugins pass it through.
    priority: int = 1
    callback: Optional[Callable] = None
    # Callbacks from submitters that JOINED this fetch instead of starting a
    # duplicate one. The primary `callback` above belongs to whoever created
    # the request; these belong to everyone who asked for the same cache_key
    # while it was still in flight.
    extra_callbacks: List[Callable] = field(default_factory=list)
    created_at: float = field(default_factory=time.time)
    status: FetchStatus = FetchStatus.PENDING
    # Set once the worker has decided this response will be cached, while it
    # still holds the lock. From that point cancelling is refused: the write
    # is already authorised, and abandoning it here would put the payload in
    # the cache with the callbacks suppressed -- joiners waiting forever for a
    # fetch that did, in fact, succeed.
    commit_claimed: bool = False
    result: Optional[Any] = None
    error: Optional[str] = None

@dataclass
class FetchResult:
    """Result of a background fetch operation.

    ``data`` survives on the stored result only for requests submitted without
    a ``callback``, where polling ``get_result()`` is the sole way to collect
    it. When a callback was given, the payload has already been delivered and
    the service releases it -- see :meth:`BackgroundDataService._release_payload`.
    Either way the data remains in the cache under the request's ``cache_key``,
    which is where consumers read it from.
    """
    request_id: str
    success: bool
    data: Optional[Any] = None
    error: Optional[str] = None
    cached: bool = False
    fetch_time: float = 0.0
    retry_count: int = 0
    completed_at: float = field(default_factory=time.time)  # Timestamp when request completed
    # The request's final status, recorded so a finished request can still be
    # reported accurately. Without it a caller can only be told COMPLETED or
    # FAILED, which turns "you cancelled this" into "this errored".
    final_status: Optional[FetchStatus] = None

class BackgroundDataService:
    """
    Background data service for fetching season data without blocking the main thread.
    
    This service manages a pool of background threads to fetch data asynchronously,
    with intelligent caching, retry logic, and progress tracking.
    """

    # Plugins feature-detect this. A core without it sends season ranges to
    # ESPN as-is and gets 400s since 2026-09-15, so plugins fetch those
    # ranges themselves instead of submitting them here.
    handles_espn_date_ranges = True

    def __init__(self, cache_manager: CacheManager, max_workers: int = 3, request_timeout: int = 30):
        """
        Initialize the background data service.
        
        Args:
            cache_manager: Cache manager instance for storing fetched data
            max_workers: Maximum number of background threads
            request_timeout: Default timeout for HTTP requests
        """
        self.cache_manager = cache_manager
        self.max_workers = max_workers
        self.request_timeout = request_timeout
        
        # Thread management
        self.executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="BackgroundData")
        # cache_key -> request_id for fetches currently in flight, so a second
        # submit for the same key joins the running fetch instead of starting
        # another. It is the normal case: a sport's Recent and Upcoming
        # managers miss the cache for the same season schedule together.
        self._inflight_by_cache_key: Dict[str, str] = {}
        # Makes every request_id unique. The id also carries a millisecond
        # timestamp, but two submits can share a millisecond, and a joiner
        # uses the id as its handle for get_result().
        self._request_seq = itertools.count()
        self.active_requests: Dict[str, FetchRequest] = {}
        self.completed_requests: Dict[str, FetchResult] = {}
        
        # Thread safety
        self._lock = threading.RLock()
        self._shutdown = False
        
        # Cleanup tracking
        self._max_completed_requests = 500  # Maximum completed requests to keep
        self._completed_requests_cleanup_interval = 600.0  # Cleanup every 10 minutes
        self._last_completed_requests_cleanup = time.time()
        
        # Statistics
        self.stats = {
            'total_requests': 0,
            'completed_requests': 0,
            'failed_requests': 0,
            'cached_hits': 0,
            'cache_misses': 0,
            'total_fetch_time': 0.0,
            'average_fetch_time': 0.0
        }
        
        # Session for HTTP requests
        self.session = requests.Session()
        self.session.mount('http://', requests.adapters.HTTPAdapter(max_retries=3))
        self.session.mount('https://', requests.adapters.HTTPAdapter(max_retries=3))
        
        # Default headers: core's shared set (real User-Agent, no hand-set
        # Accept-Encoding) -- see src/common/api_helper.py.
        from src.common.api_helper import DEFAULT_HTTP_HEADERS
        self.default_headers = dict(DEFAULT_HTTP_HEADERS)
        
        logger.info(f"BackgroundDataService initialized with {max_workers} workers")
    
    def get_sport_cache_key(self, sport: str, date_str: str = None) -> str:
        """
        Generate consistent cache keys for sports data.
        This ensures Recent/Upcoming managers and background service
        use the same cache keys.
        """
        # Same format as CacheManager.generate_sport_cache_key(), built here
        # rather than by constructing a CacheManager (config load, cache-dir
        # probing) on every submit without a cache_key.
        if date_str is None:
            date_str = datetime.now(pytz.utc).strftime('%Y%m%d')
        return f"{sport}_{date_str}"

    def submit_fetch_request(self, 
                           sport: str, 
                           year: int, 
                           url: str,
                           cache_key: str = None,
                           params: Optional[Dict[str, Any]] = None,
                           headers: Optional[Dict[str, str]] = None,
                           timeout: Optional[int] = None,
                           max_retries: int = 3,
                           priority: int = 1,
                           callback: Optional[Callable] = None) -> str:
        """
        Submit a background fetch request.
        
        Args:
            sport: Sport identifier (e.g., 'nfl', 'ncaafb')
            year: Year to fetch data for
            url: URL to fetch data from
            cache_key: Cache key for storing/retrieving data
            params: URL parameters
            headers: HTTP headers
            timeout: Request timeout
            max_retries: Maximum number of retries
            priority: Accepted for compatibility and ignored; requests run in
                submission order.
            callback: Optional callback function when request completes
            
        Returns:
            Request ID for tracking the fetch operation
        """
        if self._shutdown:
            raise RuntimeError("BackgroundDataService is shutting down")
        
        # Generate cache key if not provided
        if cache_key is None:
            cache_key = self.get_sport_cache_key(sport)
        
        with self._lock:
            request_id = (f"{sport}_{year}_{int(time.time() * 1000)}"
                          f"_{next(self._request_seq)}")
        
        # Check cache first
        cached_data = self.cache_manager.get(cache_key)
        if cached_data:
            with self._lock:
                self.stats['cached_hits'] += 1
                result = FetchResult(
                    request_id=request_id,
                    success=True,
                    data=cached_data,
                    cached=True,
                    fetch_time=0.0
                )
                # Filed before the callback runs, as it always was: a callback
                # that queries get_result()/is_request_complete() for its own
                # request must still find it. Releasing afterwards mutates the
                # same object the dict holds.
                self.completed_requests[request_id] = result

                if callback:
                    try:
                        callback(result)
                    except Exception as e:
                        logger.error(f"Error in callback for request {request_id}: {e}")
                    self._release_payload(result)

                logger.debug(f"Cache hit for {sport} {year} data")
                return request_id
        
        # limit above 500 makes an ESPN *scoreboard* return a truncated list
        # (src/common/espn_dates.py). Other endpoints need more: /teams has 762
        # college-football teams, so only scoreboards are clamped.
        if url.split('?', 1)[0].rstrip('/').endswith('/scoreboard'):
            params = clamp_espn_limit(params)

        # Create fetch request
        request = FetchRequest(
            id=request_id,
            sport=sport,
            year=year,
            cache_key=cache_key,
            url=url,
            params=dict(params or {}),
            headers={**self.default_headers, **(headers or {})},
            timeout=timeout or self.request_timeout,
            max_retries=max_retries,
            priority=priority,
            callback=callback
        )
        
        with self._lock:
            existing_id = self._inflight_by_cache_key.get(cache_key)
            existing = self.active_requests.get(existing_id) if existing_id else None
            if existing_id and existing is None:
                # Stranded index entry: the request it names is gone. Drop it and
                # fetch normally. Looking the request up rather than trusting the
                # id is what stops a stale entry wedging a key forever.
                del self._inflight_by_cache_key[cache_key]
            if existing is not None:
                # Someone is already fetching this key. Ride along rather than
                # duplicating the download, the parse and the resident copy.
                if callback:
                    existing.extra_callbacks.append(callback)
                self.stats['deduplicated_requests'] = (
                    self.stats.get('deduplicated_requests', 0) + 1
                )
                logger.info(
                    "Joined in-flight fetch %s for %s (cache_key=%s) instead of "
                    "starting a duplicate", existing_id, sport, cache_key
                )
                return existing_id

            self.active_requests[request_id] = request
            self._inflight_by_cache_key[cache_key] = request_id
            self.stats['total_requests'] += 1
            self.stats['cache_misses'] += 1
        
        # Submit to executor
        self.executor.submit(self._fetch_data_worker, request)
        
        logger.info(f"Submitted background fetch request {request_id} for {sport} {year}")
        return request_id
    
    def _fetch_data_worker(self, request: FetchRequest) -> FetchResult:
        """
        Worker function that performs the actual data fetching.
        
        Args:
            request: Fetch request to process
            
        Returns:
            Fetch result with data or error information
        """
        start_time = time.time()
        result = FetchResult(request_id=request.id, success=False, retry_count=request.retry_count)
        
        try:
            with self._lock:
                # A request cancelled while it sat in the executor queue stays
                # cancelled: no download, no cache write, no callback.
                if request.status == FetchStatus.CANCELLED:
                    cancelled_before_start = True
                else:
                    cancelled_before_start = False
                    request.status = FetchStatus.IN_PROGRESS
            if cancelled_before_start:
                logger.info(
                    "Request %s was cancelled before its worker started; "
                    "skipping the fetch entirely", request.id
                )
                # Assign before returning: the finally block stores `result`
                # in completed_requests, so building a fresh one here would
                # file the untouched placeholder instead of this outcome.
                result = FetchResult(
                    request_id=request.id,
                    success=False,
                    error="cancelled",
                    fetch_time=time.time() - start_time,
                    retry_count=request.retry_count
                )
                return result
            
            logger.info(f"Starting background fetch for {request.sport} {request.year}")
            
            # ESPN stopped accepting dates=YYYYMMDD-YYYYMMDD on 2026-09-15 and
            # answers 400 for every sport. Re-ask in months and days rather
            # than let a whole season fail. See src/common/espn_dates.py.
            # The "ranges are rejected" memo is shared with
            # fetch_espn_scoreboard(): once either path has seen a range
            # rejected, the other skips the doomed range request too.
            is_range = parse_espn_date_range(request.params.get("dates")) is not None
            data = None
            chunks_tried = False
            if is_range and _ranges_known_rejected():
                data = self._fetch_in_date_chunks(request)
                # Every chunk failed: ask for the range itself below so the
                # failure carries a real HTTP error, without re-spending chunks.
                chunks_tried = data is None

            if data is None:
                # Perform HTTP request with retry logic
                response = self._make_request_with_retry(request)
                if is_range and response.status_code == 400 and not chunks_tried:
                    _note_range_rejected()
                    logger.warning(
                        "ESPN rejected the date range %s (400); fetching it as "
                        "month/day chunks, and fetching ranges that way for the "
                        "next %d hours",
                        request.params.get("dates"), RANGE_RETRY_SECONDS // 3600,
                    )
                    data = self._fetch_in_date_chunks(request)
                    if data is None:
                        response.raise_for_status()
                else:
                    response.raise_for_status()
                    data = response_json(response)
            
            # Validate data structure
            if not isinstance(data, dict):
                raise ValueError(f"Expected dict response, got {type(data)}")
            
            if 'events' not in data:
                raise ValueError("Response missing 'events' field")
            
            # Validate events structure
            events = data.get('events', [])
            if not isinstance(events, list):
                raise ValueError(f"Expected events to be list, got {type(events)}")
            
            # Log data validation
            logger.debug(f"Validated {len(events)} events for {request.sport} {request.year}")
            
            # A cancelled request must not commit anything. Cancelling
            # releases the cache_key, so a replacement fetch for the same key
            # may already be in flight or finished -- writing this response to
            # the cache now would overwrite fresher data with the response
            # nobody wanted. The worker has no way to abort the HTTP call, so
            # this is where the work gets discarded.
            with self._lock:
                cancelled = request.status == FetchStatus.CANCELLED
                if not cancelled:
                    # Claim the commit in the same critical section that read
                    # the status, so a cancel cannot slip in between the check
                    # and the cache write below. The write itself stays outside
                    # the lock: it serialises a multi-megabyte payload to the
                    # SD card, and holding the service lock across that would
                    # stall every submit, status query and cancel behind it.
                    request.commit_claimed = True
            if cancelled:
                logger.info(
                    "Discarding response for cancelled request %s; %s may "
                    "already belong to a replacement fetch",
                    request.id, request.cache_key
                )
                result = FetchResult(
                    request_id=request.id,
                    success=False,
                    error="cancelled",
                    fetch_time=time.time() - start_time,
                    retry_count=request.retry_count
                )
                return result

            # Cache the data
            self.cache_manager.set(request.cache_key, data)
            
            # Update request status
            with self._lock:
                request.status = FetchStatus.COMPLETED
                request.result = data
            
            # Create successful result
            fetch_time = time.time() - start_time
            result = FetchResult(
                request_id=request.id,
                success=True,
                data=data,
                fetch_time=fetch_time,
                retry_count=request.retry_count
            )
            
            logger.info(f"Successfully fetched {request.sport} {request.year} data in {fetch_time:.2f}s")
            
        except Exception as e:
            error_msg = str(e)
            logger.error(f"Failed to fetch {request.sport} {request.year} data: {error_msg}")
            
            with self._lock:
                # A cancelled request stays CANCELLED even when its fetch
                # failed: the finally block skips callbacks only for
                # CANCELLED, and nobody is waiting on this fetch any more.
                if request.status != FetchStatus.CANCELLED:
                    request.status = FetchStatus.FAILED
                request.error = error_msg
            
            result = FetchResult(
                request_id=request.id,
                success=False,
                error=error_msg,
                fetch_time=time.time() - start_time,
                retry_count=request.retry_count
            )
        
        finally:
            # Store result and clean up
            with self._lock:
                result.final_status = request.status
                self.completed_requests[request.id] = result
                if request.id in self.active_requests:
                    del self.active_requests[request.id]
                # Stop accepting joiners and take the callback list in the same
                # critical section. A submitter that arrives after this point
                # finds no in-flight entry and either hits the cache (written
                # above, before the result was built) or starts a fresh fetch --
                # what it must never do is join a fetch whose callbacks have
                # already run and then never be called.
                if self._inflight_by_cache_key.get(request.cache_key) == request.id:
                    del self._inflight_by_cache_key[request.cache_key]
                # A cancelled request delivers nothing: its joiners were told
                # about a fetch that has been abandoned, and a replacement will
                # call them via its own request.
                if request.status == FetchStatus.CANCELLED:
                    callbacks = []
                else:
                    callbacks = ([request.callback] if request.callback else [])
                    callbacks.extend(request.extra_callbacks)
                
                # Update statistics
                if result.success:
                    self.stats['completed_requests'] += 1
                else:
                    self.stats['failed_requests'] += 1
                
                self.stats['total_fetch_time'] += result.fetch_time
                self.stats['average_fetch_time'] = (
                    self.stats['total_fetch_time'] / 
                    (self.stats['completed_requests'] + self.stats['failed_requests'])
                )
            
            # Periodic cleanup after storing result
            self._cleanup_completed_requests()
            
            # Call every callback: the original submitter's and any that joined
            # this fetch. One raising must not stop the others being delivered.
            for cb in callbacks:
                try:
                    cb(result)
                except Exception as e:
                    logger.error(f"Error in callback for request {request.id}: {e}")

            # Released after the loop, never inside it: every callback holds
            # the same FetchResult (a sport's recent, upcoming and live
            # managers usually share one fetch), so a release between
            # deliveries would hand the later ones `result.data is None`.
            #
            # Only when there were callbacks: a request submitted without one
            # collects its payload by polling get_result().
            if callbacks:
                self._release_payload(result)
                request.result = None

        return result

    @staticmethod
    def _release_payload(result: FetchResult) -> None:
        """Drop a delivered payload, keeping the result's status and timings.

        Only called once EVERY callback has been handed the data -- callers
        that joined an in-flight fetch share this object, so releasing between
        deliveries strips the payload out from under the ones still queued.
        Consumers read fetched data back from the cache under ``cache_key``;
        the copy carried
        here was pinning a parsed season schedule -- 946 games for NCAA
        football, roughly a tenth of total RAM on a 1GB Pi -- in memory until
        the hourly sweep.

        The cache-hit path matters most: it runs once per update interval per
        sport, mints a fresh request_id each time, and a memory-tier miss
        re-parses the payload from disk. Those were genuinely separate copies
        accumulating toward the 500-entry cap, not shared references.
        """
        result.data = None
    
    def _fetch_in_date_chunks(self, request: FetchRequest) -> Optional[Dict[str, Any]]:
        """Re-fetch a rejected ``YYYYMMDD-YYYYMMDD`` range as month/day chunks.

        None means the request was not a day range, or every chunk failed; the
        caller then re-raises the original 400 instead of caching an empty
        season. See src/common/espn_dates.py.
        """
        logger.info("Recovering %s %s from a rejected date range", request.sport, request.year)
        return fetch_espn_date_chunks(
            self.session,
            request.url,
            params=request.params,
            headers=request.headers,
            timeout=request.timeout,
            logger=logger,
        )

    def _make_request_with_retry(self, request: FetchRequest) -> requests.Response:
        """
        Make HTTP request with retry logic and exponential backoff.
        
        Args:
            request: Fetch request containing request details
            
        Returns:
            HTTP response
            
        Raises:
            requests.RequestException: If all retries fail
        """
        last_exception = None
        
        for attempt in range(request.max_retries + 1):
            try:
                response = self.session.get(
                    request.url,
                    params=request.params,
                    headers=request.headers,
                    timeout=request.timeout
                )
                return response
                
            except requests.RequestException as e:
                last_exception = e
                request.retry_count = attempt + 1
                
                if attempt < request.max_retries:
                    # Exponential backoff: 1s, 2s, 4s, 8s...
                    delay = 2 ** attempt
                    logger.warning(f"Request failed (attempt {attempt + 1}/{request.max_retries + 1}), retrying in {delay}s: {e}")
                    time.sleep(delay)
                else:
                    logger.error(f"All {request.max_retries + 1} attempts failed for {request.sport} {request.year}")
        
        raise last_exception
    
    def get_result(self, request_id: str) -> Optional[FetchResult]:
        """
        Get the result of a fetch request.
        
        Args:
            request_id: Request ID to get result for
            
        Returns:
            Fetch result if available, None otherwise
        """
        # Periodic cleanup
        self._cleanup_completed_requests()
        
        with self._lock:
            return self.completed_requests.get(request_id)
    
    def is_request_complete(self, request_id: str) -> bool:
        """
        Check if a request has completed.
        
        Args:
            request_id: Request ID to check
            
        Returns:
            True if request is complete, False otherwise
        """
        # Periodic cleanup
        self._cleanup_completed_requests()
        
        with self._lock:
            return request_id in self.completed_requests
    
    def get_request_status(self, request_id: str) -> Optional[FetchStatus]:
        """
        Get the status of a fetch request.
        
        Args:
            request_id: Request ID to get status for
            
        Returns:
            Request status if found, None otherwise
        """
        with self._lock:
            if request_id in self.active_requests:
                return self.active_requests[request_id].status
            elif request_id in self.completed_requests:
                result = self.completed_requests[request_id]
                if result.final_status is not None:
                    return result.final_status
                return FetchStatus.COMPLETED if result.success else FetchStatus.FAILED
            return None
    
    def cancel_request(self, request_id: str) -> bool:
        """
        Cancel a pending or in-progress request.
        
        Args:
            request_id: Request ID to cancel
            
        Returns:
            True if request was cancelled, False if not found or already complete
        """
        with self._lock:
            if request_id in self.active_requests:
                request = self.active_requests[request_id]
                if request.commit_claimed:
                    # Too late: the worker holds an authorised commit. Report
                    # the failure rather than half-cancelling a request whose
                    # data is about to land in the cache.
                    logger.debug(
                        "Not cancelling %s: its response is already being "
                        "committed", request_id
                    )
                    return False
                request.status = FetchStatus.CANCELLED
                del self.active_requests[request_id]
                # Cancelling is the other way a request leaves active_requests,
                # so the in-flight index has to be released here too or the key
                # stays pointed at a request that no longer exists.
                if self._inflight_by_cache_key.get(request.cache_key) == request_id:
                    del self._inflight_by_cache_key[request.cache_key]
                logger.info(f"Cancelled request {request_id}")
                return True
            return False
    
    def get_statistics(self) -> Dict[str, Any]:
        """
        Get service statistics.
        
        Returns:
            Dictionary containing service statistics
        """
        with self._lock:
            return {
                **self.stats,
                'active_requests': len(self.active_requests),
                'completed_requests_count': len(self.completed_requests),
                'max_completed_requests': self._max_completed_requests,
                'completed_requests_usage_percent': (len(self.completed_requests) / self._max_completed_requests * 100) if self._max_completed_requests > 0 else 0,
                'last_cleanup': self._last_completed_requests_cleanup,
                'cleanup_interval': self._completed_requests_cleanup_interval
            }
    
    def log_memory_stats(self):
        """Log current memory usage statistics."""
        stats = self.get_statistics()
        logger.info(f"BackgroundDataService Memory - Active: {stats['active_requests']}, "
                   f"Completed: {stats['completed_requests_count']}/{stats['max_completed_requests']} "
                   f"({stats['completed_requests_usage_percent']:.1f}%), "
                   f"Last cleanup: {time.time() - stats['last_cleanup']:.1f}s ago")
    
    def _cleanup_completed_requests(self, force: bool = False) -> int:
        """
        Automatically clean up old completed requests.
        
        Args:
            force: If True, perform cleanup regardless of time interval
            
        Returns:
            Number of requests removed
        """
        now = time.time()
        
        # Check if cleanup is needed
        if not force and (now - self._last_completed_requests_cleanup) < self._completed_requests_cleanup_interval:
            return 0
        
        with self._lock:
            removed_count = 0
            current_time = time.time()
            
            # Remove requests older than 1 hour
            cutoff_time = current_time - 3600  # 1 hour
            
            to_remove = []
            for request_id, result in self.completed_requests.items():
                # Check if request is old enough to remove
                if result.completed_at < cutoff_time:
                    to_remove.append(request_id)
            
            # Also enforce size limit if we have too many requests
            if len(self.completed_requests) > self._max_completed_requests:
                # Sort by completion time (oldest first)
                sorted_requests = sorted(
                    self.completed_requests.items(),
                    key=lambda x: x[1].completed_at
                )
                
                # Remove oldest entries until we're under the limit
                excess_count = len(self.completed_requests) - self._max_completed_requests
                for i in range(excess_count):
                    if i < len(sorted_requests):
                        request_id = sorted_requests[i][0]
                        if request_id not in to_remove:
                            to_remove.append(request_id)
            
            # Remove the requests
            for request_id in to_remove:
                del self.completed_requests[request_id]
                removed_count += 1
            
            self._last_completed_requests_cleanup = current_time
            
            if removed_count > 0:
                logger.debug(f"Cleaned up {removed_count} old completed requests (remaining: {len(self.completed_requests)})")
            
            return removed_count
    
    def shutdown(self, wait: bool = True):
        """
        Shutdown the background data service.

        Args:
            wait: Whether to wait for active requests to complete
        """
        logger.info("Shutting down BackgroundDataService...")
        
        self._shutdown = True
        
        # Cancel all active requests
        with self._lock:
            for request_id in list(self.active_requests.keys()):
                self.cancel_request(request_id)
        
        self.executor.shutdown(wait=wait)
        
        logger.info("BackgroundDataService shutdown complete")
    
    def __del__(self):
        """Cleanup when service is destroyed."""
        if not self._shutdown:
            self.shutdown(wait=False)

# Global service instance
_background_service: Optional[BackgroundDataService] = None
_service_lock = threading.Lock()

def get_background_service(cache_manager=None, max_workers: int = 3) -> BackgroundDataService:
    """
    Get the global background data service instance.
    
    Args:
        cache_manager: Cache manager instance (required for first call)
        max_workers: Maximum number of background threads
        
    Returns:
        Background data service instance
    """
    global _background_service
    
    with _service_lock:
        if _background_service is None:
            if cache_manager is None:
                raise ValueError("cache_manager is required for first call to get_background_service")
            _background_service = BackgroundDataService(cache_manager, max_workers)
        
        return _background_service

def shutdown_background_service():
    """Shutdown the global background data service."""
    global _background_service
    
    with _service_lock:
        if _background_service is not None:
            _background_service.shutdown()
            _background_service = None