From a90e6965efca005549a7f67dbbcbe771a1619424 Mon Sep 17 00:00:00 2001 From: ChuckBuilds <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 7 May 2025 09:53:36 -0500 Subject: [PATCH] fix: improve Yahoo Finance API rate limiting handling - Add proper User-Agent and headers - Increase request delays between symbols - Add 429 status code to retry list - Increase retry attempts and backoff factor --- src/stock_news_manager.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/stock_news_manager.py b/src/stock_news_manager.py index db4bd13f..d5a034c8 100644 --- a/src/stock_news_manager.py +++ b/src/stock_news_manager.py @@ -63,15 +63,20 @@ class StockNewsManager: self.rotation_complete = False # Flag to indicate when a full rotation is complete self.headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' + 'User-Agent': 'LEDMatrix/1.0 (https://github.com/yourusername/LEDMatrix; contact@example.com)', + 'Accept': 'application/json', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'Connection': 'keep-alive' } # Set up session with retry logic self.session = requests.Session() retry_strategy = Retry( - total=3, # number of retries - backoff_factor=0.5, # wait 0.5, 1, 2 seconds between retries - status_forcelist=[500, 502, 503, 504], # HTTP status codes to retry on + total=5, # increased number of retries + backoff_factor=1, # increased backoff factor + status_forcelist=[429, 500, 502, 503, 504], # added 429 to retry list + allowed_methods=["GET", "HEAD", "OPTIONS"] ) adapter = HTTPAdapter(max_retries=retry_strategy) self.session.mount("https://", adapter) @@ -173,8 +178,8 @@ class StockNewsManager: success = True continue - # Add a small delay between requests to avoid rate limiting - time.sleep(random.uniform(0.1, 0.3)) + # Add a longer delay between requests to avoid rate limiting + time.sleep(random.uniform(1.0, 2.0)) # increased delay between requests news_items = self._fetch_news(symbol) if news_items: new_data[symbol] = news_items