From b210fb60832959c7d10a423ae247e4fcfc24e77d Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sun, 4 Oct 2026 17:19:43 -0400 Subject: [PATCH] feat(fetch): count bytes on the wire as well as decoded fetch-stats reported only `bytes`, len(response.content), and that read as the download volume. ESPN gzips every scoreboard, so it overstated real traffic about 14x: a college football Saturday is 865 KB decoded, 63 KB on the wire. Every counter set now carries `wire_bytes`, read from urllib3's count of raw bytes taken off the socket (decoded size when there is no urllib3 response behind it). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 ++++++++++++ src/common/fetch_service.py | 31 ++++++++++++++++++++++++++++++- test/test_fetch_service.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cc6a860..33434637 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,18 @@ soccer-scoreboard 2.39.2, alternating runs: **~450 requests per start, peak spends one doomed 400 per window at every start (eleven at once from a soccer board); the range is still retried `RANGE_RETRY_SECONDS` in. +### Fetch stats: bytes on the wire, not just decoded + +`GET /api/v3/plugins/fetch-stats` reported only `bytes`, the decoded body +size, and that read as the download volume. ESPN gzips every scoreboard, so +it overstated what crossed the network about 14x: a college football +Saturday's scoreboard is 865 KB decoded and 63 KB on the wire, and ledpi's +"643 MB in 6 hours" of football was ~47 MB of actual traffic. Every counter +set (totals, per plugin, per host) now has `wire_bytes` too, read from +urllib3's count of the raw bytes it took off the socket. A response with no +urllib3 response behind it is counted at its decoded size. `bytes` keeps its +meaning. + ### Cheap per-frame and per-fetch savings - `BaseOddsManager.get_odds()` no longer pretty-prints every odds response diff --git a/src/common/fetch_service.py b/src/common/fetch_service.py index c116776b..0f86e7cc 100644 --- a/src/common/fetch_service.py +++ b/src/common/fetch_service.py @@ -59,7 +59,10 @@ says how old with ``cache_max_age`` (``fetch_get(..., cache_max_age=ttl)``; Identical means what the validator store keys on: URL, query, effective headers and, for a session with cookies or auth, the session. -**Counters.** Requests, merged requests, bytes, 304s, errors, HTTP errors, +**Counters.** Requests, merged requests, bytes (``bytes`` decoded, as the +caller reads them; ``wire_bytes`` as they crossed the network, which is +what a metered connection pays for -- ESPN gzips, so the two differ ~14x), +304s, errors, HTTP errors, adapter retries, throttled requests and seconds waited, plus requests answered without the network: ``memo_hits`` (the response cache) and ``cache_hits`` / ``legacy_cache_hits`` (a shared ESPN scoreboard cache entry, @@ -201,6 +204,7 @@ _COUNTER_FIELDS = ( "throttled", # requests that waited for a host budget "overruns", # requests that went after max_wait_seconds anyway "bytes", # decoded response body bytes received + "wire_bytes", # body bytes as they came off the socket (still compressed) "wait_seconds", # time spent waiting for host budgets "memo_hits", # answered from the response cache (max-age); nothing sent "cache_hits", # scoreboard fetches answered from a shared ESPN cache entry @@ -616,6 +620,30 @@ def _body_of(response: Any) -> Optional[bytes]: return content if isinstance(content, bytes) else None +def _wire_bytes_of(response: Any, body: Optional[bytes]) -> int: + """How many body bytes came off the socket for ``response``: the + compressed size when the server sent gzip, which ESPN does for every + scoreboard (63 KB on the wire for an 865 KB college football Saturday). + + urllib3's ``HTTPResponse.tell()`` counts the raw bytes read before + decoding. A response without one (a test double, an adapter that is not + urllib3) or one whose body was not read is counted at its decoded size, + or as 0, so the counter never claims less than it can prove. + """ + if body is None: + return 0 + raw = getattr(response, "raw", None) + tell = getattr(raw, "tell", None) + if callable(tell): + try: + read = tell() + except Exception: + read = None + if isinstance(read, int) and not isinstance(read, bool) and read > 0: + return read + return len(body) + + def _retries_of(response: Any) -> int: raw = getattr(response, "raw", None) retries = getattr(raw, "retries", None) @@ -1117,6 +1145,7 @@ class FetchService: http_errors=int(status is not None and status >= 400), retries=_retries_of(response), bytes=len(body) if body is not None else 0, + wire_bytes=_wire_bytes_of(response, body), throttled=int(waited > 0), overruns=int(overrun), wait_seconds=waited) except Exception: diff --git a/test/test_fetch_service.py b/test/test_fetch_service.py index d2c17f4c..22a4231e 100644 --- a/test/test_fetch_service.py +++ b/test/test_fetch_service.py @@ -576,6 +576,42 @@ class TestCounters: assert snap["hosts"]["site.api.espn.com"]["requests"] == 1 assert snap["totals"]["bytes"] == 3 * len(b'{"ok": 1}') + def test_wire_bytes_are_the_compressed_size(self, service): + # Built the way requests builds a real response: a urllib3 + # HTTPResponse carrying a gzip body, decoded when .content is read. + import gzip + import io + + from requests.adapters import HTTPAdapter + from urllib3.response import HTTPResponse + + decoded = json.dumps({"events": [{"id": str(i), "name": "x" * 200} + for i in range(50)]}).encode() + wire = gzip.compress(decoded) + + def handler(url, kwargs): + raw = HTTPResponse(body=io.BytesIO(wire), status=200, + headers={"Content-Encoding": "gzip", + "Content-Type": "application/json"}, + preload_content=False, decode_content=True) + request = requests.Request("GET", url).prepare() + response = HTTPAdapter().build_response(request, raw) + response.content # what Session.get does for a non-streamed call + return response + + response = service.get(FakeSession(handler), "https://site.api.espn.com/x") + assert response.content == decoded + totals = _counters(service) + assert totals["bytes"] == len(decoded) + assert totals["wire_bytes"] == len(wire) < len(decoded) + + def test_wire_bytes_fall_back_to_the_decoded_size(self, service): + # No urllib3 response behind it (a test double, another adapter): + # count what is known rather than nothing. + service.get(FakeSession(), "https://api.test/x") + totals = _counters(service) + assert totals["wire_bytes"] == totals["bytes"] == len(b'{"ok": 1}') + def test_errors_and_http_errors(self, service): def handler(url, kwargs): if url.endswith("/down"):