Feature/memory optimization config (#108)

* feat(config): optimize memory usage to prevent OOM killer

- Reduce brightness from 95 to 50 to lower power consumption
- Reduce refresh rate from 120Hz to 100Hz to reduce CPU/memory pressure
- Reduce background service workers from 3 to 1 per manager
- Change hardware mapping from adafruit-hat-pwm to adafruit-hat
- Expected memory savings: ~700MB reduction in background service usage
- Addresses SIGKILL errors caused by memory exhaustion on Raspberry Pi

Fixes: OOM killer terminating ledmatrix.service with status=9/KILL

* revert display brightness

* refactor(background-service): hardcode optimized settings and remove config blocks

- Hardcode background service to 1 worker in all managers
- Remove background_service config blocks from template
- Simplify configuration for users - no need to adjust system settings
- Memory optimization: ~700MB reduction in background service usage
- Settings: 1 worker, 30s timeout, 3 retries (hardcoded)

Files updated:
- src/base_classes/sports.py
- src/leaderboard_manager.py
- src/odds_ticker_manager.py
- src/soccer_managers.py
- src/milb_manager.py
- config/config.template.json

This prevents OOM killer errors by reducing memory usage from
15 background threads to 5 threads total across all managers.

* remove fallback in case of background service failure
This commit is contained in:
Chuck
2025-10-08 20:10:54 -04:00
committed by GitHub
parent 58bbb5fe6f
commit 7a61ecff7b
18 changed files with 36 additions and 437 deletions

View File

@@ -105,19 +105,12 @@ class SportsCore(ABC):
self._rankings_cache_timestamp = 0
self._rankings_cache_duration = 3600 # Cache rankings for 1 hour
# Initialize background data service
background_config = self.mode_config.get("background_service", {})
if background_config.get("enabled", True): # Default to enabled
max_workers = background_config.get("max_workers", 3)
self.background_service = get_background_service(self.cache_manager, max_workers)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
self.logger.info(f"Background service enabled with {max_workers} workers")
else:
self.background_service = None
self.background_fetch_requests = {}
self.background_enabled = False
self.logger.info("Background service disabled")
# Initialize background data service with optimized settings
# Hardcoded for memory optimization: 1 worker, 30s timeout, 3 retries
self.background_service = get_background_service(self.cache_manager, max_workers=1)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
self.logger.info("Background service enabled with 1 worker (memory optimized)")
def _get_season_schedule_dates(self) -> tuple[str, str]:
return "", ""

View File

@@ -71,19 +71,12 @@ class LeaderboardManager:
# Store reference to config instead of creating new ConfigManager
self.config = config
# Initialize background data service
background_config = self.leaderboard_config.get("background_service", {})
if background_config.get("enabled", True): # Default to enabled
max_workers = background_config.get("max_workers", 3)
self.background_service = get_background_service(self.cache_manager, max_workers)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
logger.info(f"[Leaderboard] Background service enabled with {max_workers} workers")
else:
self.background_service = None
self.background_fetch_requests = {}
self.background_enabled = False
logger.info("[Leaderboard] Background service disabled")
# Initialize background data service with optimized settings
# Hardcoded for memory optimization: 1 worker, 30s timeout, 3 retries
self.background_service = get_background_service(self.cache_manager, max_workers=1)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
logger.info("[Leaderboard] Background service enabled with 1 worker (memory optimized)")
# State variables
self.last_update = 0

View File

@@ -75,19 +75,12 @@ class BaseMiLBManager(Baseball):
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
# Initialize background data service
background_config = self.milb_config.get("background_service", {})
if background_config.get("enabled", True): # Default to enabled
max_workers = background_config.get("max_workers", 3)
self.background_service = get_background_service(self.cache_manager, max_workers)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
self.logger.info(f"[MiLB] Background service enabled with {max_workers} workers")
else:
self.background_service = None
self.background_fetch_requests = {}
self.background_enabled = False
self.logger.info("[MiLB] Background service disabled")
# Initialize background data service with optimized settings
# Hardcoded for memory optimization: 1 worker, 30s timeout, 3 retries
self.background_service = get_background_service(self.cache_manager, max_workers=1)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
self.logger.info("[MiLB] Background service enabled with 1 worker (memory optimized)")
def _probe_and_update_from_live_feed(self, game_pk: str, game_data: Dict[str, Any]) -> bool:
"""Probe MLB Stats live feed for a game and update game_data in-place if live.

View File

@@ -82,11 +82,6 @@ class BaseMLBManager(Baseball):
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
pass
# return self._fetch_ncaa_api_data_sync(use_cache)
self.logger.info(f"Fetching full {datestring} season schedule from ESPN API...")
# Start background fetch

View File

@@ -76,10 +76,6 @@ class BaseNBAManager(Basketball):
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
return self._fetch_nba_api_data_sync(use_cache)
# Start background fetch
self.logger.info(f"Starting background fetch for {season_year} season schedule...")
@@ -123,30 +119,6 @@ class BaseNBAManager(Basketball):
return partial_data
return None
def _fetch_nba_api_data_sync(self, use_cache: bool = True) -> Optional[Dict]:
"""
Synchronous fallback for fetching NFL data when background service is disabled.
"""
now = datetime.now(pytz.utc)
current_year = now.year
cache_key = f"nba_schedule_{current_year}"
self.logger.info(f"Fetching full {current_year} season schedule from ESPN API (sync mode)...")
try:
response = self.session.get(ESPN_NBA_SCOREBOARD_URL, params={"dates": current_year, "limit":1000}, headers=self.headers, timeout=15)
response.raise_for_status()
data = response.json()
events = data.get('events', [])
if use_cache:
self.cache_manager.set(cache_key, events)
self.logger.info(f"Successfully fetched {len(events)} events for the {current_year} season.")
return {'events': events}
except requests.exceptions.RequestException as e:
self.logger.error(f"API error fetching full schedule: {e}")
return None
def _fetch_data(self) -> Optional[Dict]:
"""Fetch data using shared data mechanism or direct fetch for live."""

View File

@@ -60,11 +60,6 @@ class BaseNCAABaseballManager(Baseball):
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
pass
# return self._fetch_ncaa_api_data_sync(use_cache)
self.logger.info(f"Fetching full {datestring} season schedule from ESPN API...")
# Start background fetch

View File

@@ -73,10 +73,6 @@ class BaseNCAAFBManager(Football): # Renamed class
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
return self._fetch_ncaa_api_data_sync(use_cache)
self.logger.info(f"Fetching full {season_year} season schedule from ESPN API...")
# Start background fetch
@@ -121,30 +117,6 @@ class BaseNCAAFBManager(Football): # Renamed class
if partial_data:
return partial_data
return None
def _fetch_ncaa_api_data_sync(self, use_cache: bool = True) -> Optional[Dict]:
"""
Synchronous fallback for fetching NFL data when background service is disabled.
"""
now = datetime.now(pytz.utc)
current_year = now.year
cache_key = f"ncaafb_schedule_{current_year}"
self.logger.info(f"Fetching full {current_year} season schedule from ESPN API (sync mode)...")
try:
response = self.session.get(ESPN_NCAAFB_SCOREBOARD_URL, params={"dates": current_year, "limit":1000}, headers=self.headers, timeout=15)
response.raise_for_status()
data = response.json()
events = data.get('events', [])
if use_cache:
self.cache_manager.set(cache_key, events)
self.logger.info(f"Successfully fetched {len(events)} events for the {current_year} season.")
return {'events': events}
except requests.exceptions.RequestException as e:
self.logger.error(f"[API error fetching full schedule: {e}")
return None
def _fetch_data(self) -> Optional[Dict]:
"""Fetch data using shared data mechanism or direct fetch for live."""

View File

@@ -97,10 +97,6 @@ class BaseNCAAMBasketballManager(Basketball):
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
return self._fetch_ncaam_basketball_api_data_sync(use_cache)
# Start background fetch
self.logger.info(
f"Starting background fetch for {season_year} season schedule..."
@@ -151,41 +147,6 @@ class BaseNCAAMBasketballManager(Basketball):
return None
def _fetch_ncaam_basketball_api_data_sync(
self, use_cache: bool = True
) -> Optional[Dict]:
"""
Synchronous fallback for fetching NFL data when background service is disabled.
"""
now = datetime.now(pytz.utc)
current_year = now.year
cache_key = f"ncaa_mens_basketball_schedule_{current_year}"
self.logger.info(
f"Fetching full {current_year} season schedule from ESPN API (sync mode)..."
)
try:
response = self.session.get(
ESPN_NCAAMB_SCOREBOARD_URL,
params={"dates": current_year, "limit": 1000},
headers=self.headers,
timeout=15,
)
response.raise_for_status()
data = response.json()
events = data.get("events", [])
if use_cache:
self.cache_manager.set(cache_key, events)
self.logger.info(
f"Successfully fetched {len(events)} events for the {current_year} season."
)
return {"events": events}
except requests.exceptions.RequestException as e:
self.logger.error(f"API error fetching full schedule: {e}")
return None
def _fetch_data(self) -> Optional[Dict]:
"""Fetch data using shared data mechanism or direct fetch for live."""
if isinstance(self, NCAAMBasketballLiveManager):

View File

@@ -77,10 +77,6 @@ class BaseNCAAMHockeyManager(Hockey): # Renamed class
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
return self._fetch_ncaa_api_data_sync(use_cache)
self.logger.info(f"Fetching full {season_year} season schedule from ESPN API...")
# Start background fetch
@@ -126,31 +122,6 @@ class BaseNCAAMHockeyManager(Hockey): # Renamed class
return partial_data
return None
def _fetch_ncaa_api_data_sync(self, use_cache: bool = True) -> Optional[Dict]:
"""
Synchronous fallback for fetching NCAA Mens Hockey data when background service is disabled.
"""
now = datetime.now(pytz.utc)
current_year = now.year
cache_key = f"ncaa_mens_hockey_schedule_{current_year}"
self.logger.info(f"Fetching full {current_year} season schedule from ESPN API (sync mode)...")
try:
response = self.session.get(ESPN_NCAAMH_SCOREBOARD_URL, params={"dates": current_year, "limit":1000}, headers=self.headers, timeout=15)
response.raise_for_status()
data = response.json()
events = data.get('events', [])
if use_cache:
self.cache_manager.set(cache_key, events)
self.logger.info(f"Successfully fetched {len(events)} events for the {current_year} season.")
return {'events': events}
except requests.exceptions.RequestException as e:
self.logger.error(f"[API error fetching full schedule: {e}")
return None
def _fetch_data(self) -> Optional[Dict]:
"""Fetch data using shared data mechanism or direct fetch for live."""
if isinstance(self, NCAAMHockeyLiveManager):

View File

@@ -97,10 +97,6 @@ class BaseNCAAWBasketballManager(Basketball):
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
return self._fetch_ncaaw_basketball_api_data_sync(use_cache)
# Start background fetch
self.logger.info(
f"Starting background fetch for {season_year} season schedule..."
@@ -151,41 +147,6 @@ class BaseNCAAWBasketballManager(Basketball):
return None
def _fetch_ncaaw_basketball_api_data_sync(
self, use_cache: bool = True
) -> Optional[Dict]:
"""
Synchronous fallback for fetching NFL data when background service is disabled.
"""
now = datetime.now(pytz.utc)
current_year = now.year
cache_key = f"ncaa_womens_basketball_schedule_{current_year}"
self.logger.info(
f"Fetching full {current_year} season schedule from ESPN API (sync mode)..."
)
try:
response = self.session.get(
ESPN_NCAAWB_SCOREBOARD_URL,
params={"dates": current_year, "limit": 1000},
headers=self.headers,
timeout=15,
)
response.raise_for_status()
data = response.json()
events = data.get("events", [])
if use_cache:
self.cache_manager.set(cache_key, events)
self.logger.info(
f"Successfully fetched {len(events)} events for the {current_year} season."
)
return {"events": events}
except requests.exceptions.RequestException as e:
self.logger.error(f"API error fetching full schedule: {e}")
return None
def _fetch_data(self) -> Optional[Dict]:
"""Fetch data using shared data mechanism or direct fetch for live."""
if isinstance(self, NCAAWBasketballLiveManager):

View File

@@ -92,10 +92,6 @@ class BaseNCAAWHockeyManager(Hockey): # Renamed class
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
return self._fetch_ncaa_api_data_sync(use_cache)
self.logger.info(
f"Fetching full {season_year} season schedule from ESPN API..."
)
@@ -149,39 +145,6 @@ class BaseNCAAWHockeyManager(Hockey): # Renamed class
return partial_data
return None
def _fetch_ncaa_api_data_sync(self, use_cache: bool = True) -> Optional[Dict]:
"""
Synchronous fallback for fetching NCAA Womens Hockey data when background service is disabled.
"""
now = datetime.now(pytz.utc)
current_year = now.year
cache_key = f"ncaa_womens_hockey_schedule_{current_year}"
self.logger.info(
f"Fetching full {current_year} season schedule from ESPN API (sync mode)..."
)
try:
response = self.session.get(
ESPN_NCAAWH_SCOREBOARD_URL,
params={"dates": current_year, "limit": 1000},
headers=self.headers,
timeout=15,
)
response.raise_for_status()
data = response.json()
events = data.get("events", [])
if use_cache:
self.cache_manager.set(cache_key, events)
self.logger.info(
f"Successfully fetched {len(events)} events for the {current_year} season."
)
return {"events": events}
except requests.exceptions.RequestException as e:
self.logger.error(f"[API error fetching full schedule: {e}")
return None
def _fetch_data(self) -> Optional[Dict]:
"""Fetch data using shared data mechanism or direct fetch for live."""
if isinstance(self, NCAAWHockeyLiveManager):

View File

@@ -68,10 +68,6 @@ class BaseNFLManager(Football): # Renamed class
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
return self._fetch_nfl_api_data_sync(use_cache)
# Start background fetch
self.logger.info(f"Starting background fetch for {season_year} season schedule...")
@@ -115,30 +111,6 @@ class BaseNFLManager(Football): # Renamed class
return partial_data
return None
def _fetch_nfl_api_data_sync(self, use_cache: bool = True) -> Optional[Dict]:
"""
Synchronous fallback for fetching NFL data when background service is disabled.
"""
now = datetime.now(pytz.utc)
current_year = now.year
cache_key = f"nfl_schedule_{current_year}"
self.logger.info(f"Fetching full {current_year} season schedule from ESPN API (sync mode)...")
try:
response = self.session.get(ESPN_NFL_SCOREBOARD_URL, params={"dates": current_year, "limit":1000}, headers=self.headers, timeout=15)
response.raise_for_status()
data = response.json()
events = data.get('events', [])
if use_cache:
self.cache_manager.set(cache_key, events)
self.logger.info(f"Successfully fetched {len(events)} events for the {current_year} season.")
return {'events': events}
except requests.exceptions.RequestException as e:
self.logger.error(f"API error fetching full schedule: {e}")
return None
def _fetch_data(self) -> Optional[Dict]:
"""Fetch data using shared data mechanism or direct fetch for live."""

View File

@@ -77,10 +77,6 @@ class BaseNHLManager(Hockey):
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
return self._fetch_nhl_api_data_sync(use_cache)
# Start background fetch
self.logger.info(f"Starting background fetch for {season_year} season schedule...")
@@ -124,30 +120,6 @@ class BaseNHLManager(Hockey):
return partial_data
return None
def _fetch_nhl_api_data_sync(self, use_cache: bool = True) -> Optional[Dict]:
"""
Synchronous fallback for fetching NFL data when background service is disabled.
"""
now = datetime.now(pytz.utc)
current_year = now.year
cache_key = f"nhl_schedule_{current_year}"
self.logger.info(f"Fetching full {current_year} season schedule from ESPN API (sync mode)...")
try:
response = self.session.get(ESPN_NHL_SCOREBOARD_URL, params={"dates": current_year, "limit":1000}, headers=self.headers, timeout=15)
response.raise_for_status()
data = response.json()
events = data.get('events', [])
if use_cache:
self.cache_manager.set(cache_key, events)
self.logger.info(f"Successfully fetched {len(events)} events for the {current_year} season.")
return {'events': events}
except requests.exceptions.RequestException as e:
self.logger.error(f"API error fetching full schedule: {e}")
return None
def _fetch_data(self, date_str: str = None) -> Optional[Dict]:
"""Fetch data using shared data mechanism or direct fetch for live."""

View File

@@ -113,19 +113,12 @@ class OddsTickerManager:
# OddsManager doesn't actually use the config_manager parameter, so pass None
self.odds_manager = OddsManager(self.cache_manager, None)
# Initialize background data service
background_config = self.odds_ticker_config.get("background_service", {})
if background_config.get("enabled", True): # Default to enabled
max_workers = background_config.get("max_workers", 3)
self.background_service = get_background_service(self.cache_manager, max_workers)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
logger.info(f"[Odds Ticker] Background service enabled with {max_workers} workers")
else:
self.background_service = None
self.background_fetch_requests = {}
self.background_enabled = False
logger.info("[Odds Ticker] Background service disabled")
# Initialize background data service with optimized settings
# Hardcoded for memory optimization: 1 worker, 30s timeout, 3 retries
self.background_service = get_background_service(self.cache_manager, max_workers=1)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
logger.info("[Odds Ticker] Background service enabled with 1 worker (memory optimized)")
# State variables
self.last_update = 0

View File

@@ -93,19 +93,12 @@ class BaseSoccerManager:
self._logo_cache = {}
# Initialize background data service
background_config = self.soccer_config.get("background_service", {})
if background_config.get("enabled", True): # Default to enabled
max_workers = background_config.get("max_workers", 3)
self.background_service = get_background_service(self.cache_manager, max_workers)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
self.logger.info(f"[Soccer] Background service enabled with {max_workers} workers")
else:
self.background_service = None
self.background_fetch_requests = {}
self.background_enabled = False
self.logger.info("[Soccer] Background service disabled")
# Initialize background data service with optimized settings
# Hardcoded for memory optimization: 1 worker, 30s timeout, 3 retries
self.background_service = get_background_service(self.cache_manager, max_workers=1)
self.background_fetch_requests = {} # Track background fetch requests
self.background_enabled = True
self.logger.info("[Soccer] Background service enabled with 1 worker (memory optimized)")
# Ensure data directory exists
os.makedirs(os.path.dirname(self.team_map_file), exist_ok=True)

View File

@@ -101,10 +101,6 @@ class BaseWNBAManager(Basketball):
# Clear invalid cache
self.cache_manager.clear_cache(cache_key)
# If background service is disabled, fall back to synchronous fetch
if not self.background_enabled or not self.background_service:
return self._fetch_wnba_api_data_sync(use_cache)
# Start background fetch
self.logger.info(
f"Starting background fetch for {season_year} season schedule..."
@@ -155,39 +151,6 @@ class BaseWNBAManager(Basketball):
return None
def _fetch_wnba_api_data_sync(self, use_cache: bool = True) -> Optional[Dict]:
"""
Synchronous fallback for fetching WNBA data when background service is disabled.
"""
now = datetime.now(pytz.utc)
current_year = now.year
cache_key = f"nba_schedule_{current_year}"
self.logger.info(
f"Fetching full {current_year} season schedule from ESPN API (sync mode)..."
)
try:
response = self.session.get(
ESPN_WNBA_SCOREBOARD_URL,
params={"dates": current_year, "limit": 1000},
headers=self.headers,
timeout=15,
)
response.raise_for_status()
data = response.json()
events = data.get("events", [])
if use_cache:
self.cache_manager.set(cache_key, events)
self.logger.info(
f"Successfully fetched {len(events)} events for the {current_year} season."
)
return {"events": events}
except requests.exceptions.RequestException as e:
self.logger.error(f"API error fetching full schedule: {e}")
return None
def _fetch_data(self) -> Optional[Dict]:
"""Fetch data using shared data mechanism or direct fetch for live."""
if isinstance(self, WNBALiveManager):